{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CIFAR 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.conv_learner import *\n",
    "PATH = \"data/cifar10/\"\n",
    "os.makedirs(PATH,exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "stats = (np.array([ 0.4914 ,  0.48216,  0.44653]), np.array([ 0.24703,  0.24349,  0.26159]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(sz,bs):\n",
    "    tfms = tfms_from_stats(stats, sz, aug_tfms=[RandomFlip()], pad=sz//8)\n",
    "    return ImageClassifierData.from_paths(PATH, val_name='test', tfms=tfms, bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs=128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Look at data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "data = get_data(32,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x,y=next(iter(data.trn_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG3RJREFUeJztnVusXGd1x/9rz+XcfU58iXNwTG4EkZSWBB1ZqYgQBYFShBSQqggeUB4ijCoiFYk+RKlUgtqHUBUQDxWVaSJCRQkpARFVUUuaokZ5CTghOE7cQkhsYsf2ie/nfmb2Xn2YCT129n+dObcZO9//J1me86359l7zzV5z+f6z1jJ3hxAiPbJeOyCE6A0KfiESRcEvRKIo+IVIFAW/EImi4BciURT8QiSKgl+IRFHwC5Eo1bVMNrPbAHwTQAXAP7n7/dH9R0YGfdvW0VLban5pmGW24jkAEJ3JbDXHXP9fSYZerMrFwMdVPWag8GLFNi/4nFWz7svP1yO8Pnx167ieV9zJk1OYmp7r6JCrDn4zqwD4BwAfBXAYwC/M7DF3f4nN2bZ1FH9z352ltmazQc/Fgrxer0f+UVseBEKlWuE24od7TudEQRddRxa8sGVZFMjEjYL7GF7QwWfDuXyO2ubnZ0rHF+YX6JxitS8MUdCRQ2bBA8usxm0Zvz7g3BZFY9XLfYneEJntK/c/GpzpfNbysX8XgJfd/RV3XwTwMIDb13A8IUQXWUvw7wDw2pK/D7fHhBCXABu+4Wdmu81sr5ntPTc1u9GnE0J0yFqC/wiAnUv+vrI9dh7uvsfdJ9x9YtPI4BpOJ4RYT9YS/L8AcL2ZXWNmdQCfBvDY+rglhNhoVr3b7+5NM7sbwH+gJfU96O4vLjMHzaJ8Vz8H340u2KZnHkkyq3tdK3K+45wzR8LN92gnPVAdAv+rGODHJDvmeTFPp+S14DFn3La4yBWaxWb589kIVIdwtz9SRoLd/ozsirfEqnI8OJlHykLwnEU794tEFrXgusqoH53rnmvS+d39cQCPr+UYQojeoF/4CZEoCn4hEkXBL0SiKPiFSBQFvxCJsqbd/pXiKNDMyxM7PMgQY0kueR7m5wUm/ppnoW5HMtVC+Yc/riHwHz2N2Ti1nTk+RW2TJ8ptzWKRzunfytdjYAt/bM2MHzMHeZ4jSZdl4QAooqzEYP2Z1NoXXR/BtRhJaRb4H11W7KFF8iC7rFaSHKt3fiESRcEvRKIo+IVIFAW/EImi4BciUbq72++ORrN8FziqB8c256tVXm4pStyIcx+CHVZ6SO5HPdjRtyk+7zdH35Id/XsOHz5GbVOL5f6PDI3ROaOLfdQ2f4avY3UkSAjqK08kyms8wSja7Y92sYuwjFf5xRPVf6wH10AR7vZHtgiiZgWJTiwBzVeQ2KN3fiESRcEvRKIo+IVIFAW/EImi4BciURT8QiRKl6W+AouNcqkvj2ruERmwXuECSiU6XJi8E3S2aZYf9Nhr5d1pAGD2FE/CqU7zUuZHJ09QW2G8U9HY2HC5oXmGH6/gNQGn5/m5/Ax/3BUiXw3sHKFzbPgstTWbgRQcyoCkpmEgD+ZZ1M0nkiNXnmDUshE/gnqHzbxJfJDUJ4RYBgW/EImi4BciURT8QiSKgl+IRFHwC5Eoa5L6zOwggCkAOYCmu09E9y8cmCcSBavTBwBG6r41gxp+FrTCCsu3ebl/AJCR1lWbtvFadvV+nhXXP7uN2rZfcSW1RXXwfO5c+TjJbgOA2aD03Ow8lyNnTnA5sjZ3unR85NwQnTP4zqAN2dgcNeVV/pyx+n5RFmlzlXX6CtpXbpm2bcTHZtTarLl2qW89dP4/cXd+FQghLkr0sV+IRFlr8DuAn5rZs2a2ez0cEkJ0h7V+7L/V3Y+Y2eUAnjCz/3H3p5beof2isBsAxi7jVW2EEN1lTe/87n6k/f8kgB8D2FVynz3uPuHuE0ND/Ws5nRBiHVl18JvZkJmNvHkbwMcA7F8vx4QQG8taPvZvB/BjM3vzOP/i7v8eTXA4GkS+yAIpJCOZduGcKMMq6q4VpIg5kfpqQ3zOwCZuu2rwKmqrze+gtpNv/I7aFmfKv1rNOP/KRRItW7Y3eLHQ5jzP6luYKc/QKxrTdM7rh/labb5qE7UNX8kvYxstv3by4HmO5DwHlxWjQqLmQUFZoug5KdIJxLJip6w6+N39FQDvW7MHQoieIKlPiERR8AuRKAp+IRJFwS9Eoij4hUiUrhbwbIl95bpGVPzQifQSZrcFvfosKBYaZhcSmweyy1wgbS1mvLjn+HbeW69vhPfWO3PieOm4nzxJ58xP88KZ/c576zWCYpbTC+WZjlNn+fH6avxyPHeIP59juJzaivlyP5o8oRJF8Fu0oL4rPOitlwVZlUzSs+C6YmdaiQCod34hEkXBL0SiKPiFSBQFvxCJouAXIlG6vNsPsCyGaJeSJdu4B7v2QfZOtKMfmACiEqxWPegb5NvKo6ztFoDp6fI6fQBgeflu+uLUJJ2zOMWPNxT0PVuo8R14z8ovrcL4JZfV+HosFA1qO32aKxn1U8RwgvteezdPIqoM8cQej5SA4H2WXT8W1eNj9f1WsN2vd34hEkXBL0SiKPiFSBQFvxCJouAXIlEU/EIkSpelPkNGJKBIo2BqWRbIeZGtEs6jJjipGVgJ5CtHjdqaTV48b2GBJ8BUgwSYolKe9DMbdLSab3Dj9DSv07c4z/3v7y+vGdiY5W235uf4ubIqf5+aOs0TpIwkjNm5Op3TF7SB23TdFmqLrrk8kO1q9fLH5hUubxbNoMdah+idX4hEUfALkSgKfiESRcEvRKIo+IVIFAW/EImyrNRnZg8C+ASASXd/b3tsM4AfALgawEEAd7j76WWPBaCK8mwq1grr9xNLqATZdNWglVeF+AAAQVk6sDKDFsiU7lzqmyUtrQBgYZDLV1ng/9xcuUTYCKShZs7T0aaagUYYrNXsbLkMePIszyAcqK9Ogq0P8zXOquW2hQaX0c69+Dq1jR3mcuTlW0eprRl066r2l4dhNsgnVYfKx6N6km85fgf3+Q6A2y4YuwfAk+5+PYAn238LIS4hlg1+d38KwIVZ0bcDeKh9+yEAn1xnv4QQG8xqv/Nvd/ej7dvH0OrYK4S4hFjzhp+7O4Lf5prZbjPba2Z7Z2aCXtBCiK6y2uA/bmbjAND+n9aIcvc97j7h7hNDQ7zZhBCiu6w2+B8DcGf79p0AfrI+7gghukUnUt/3AXwIwFYzOwzgywDuB/CImd0F4BCAOzo5mQHIyDcEiwocEk0pkuWyoLhnJbBRXRFAToomRuJKVvAlPjPDqksCJ6pcbqoUXFLKSaHLep1nsUUy4LlTXMGtD5Vn7gFArb/8fCNjm+mcZs5lxcFBom0BmMt5puDUVLk0lweZewMkyw4ANm3m8tvwtdzHPLjkFqfKvw7PnuWyYn2x/IAFaf1VxrLB7+6fIaaPdHwWIcRFh37hJ0SiKPiFSBQFvxCJouAXIlEU/EIkStd79TEhLRTfmCoTFEUsnEseXnBbFkqO7HxBX0Dn0lAjkLbeOPMatW3pD7ISK+W2+aCP3EKgVWaktyIAzJ3lUuXIYLkM2BfIaEO1EWorgszDap1LjoPN8gc3PMzXcOcNl1PbZddyG4i8CQBFk/ufjy2Wjg9v40VcF+fK5cGsGjWbvOC+Hd9TCPG2QsEvRKIo+IVIFAW/EImi4BciURT8QiRKd6U+A4xVYuxcoVhCIOd50OeMVeJcDjKvEixjJDlGhT9PNY5TW73KpS2wLMI8KKTi3DYyxuW3+bNn+CEXyzPSPC+XtQAgq/HnZTCQ0abOcMlxy/Zy/9/5hzvpnKHtm6gtypnLguzIKAO1IBd/lnE5slYtf54t6Bf4luN3fE8hxNsKBb8QiaLgFyJRFPxCJIqCX4hE6epuvwPISaKIBdv9GcnssWALNcj5CWvuRduyfON+la+hwQ5wXvAd+BOzh6htgNT366/zJKKhQd7uCtZPTRXwWoILM+XtxqqB+jE7M01t3sd93Hr1ZdQ2/p4ryv0Y5YpJo8GTcCrRbnrQPs6CC5IlrlnQeivLys8VxdFbjtHxPYUQbysU/EIkioJfiERR8AuRKAp+IRJFwS9EonTSrutBAJ8AMOnu722P3QfgcwDeaN/tXnd/fPnTFchJwk0WSCjMUhRcvvKgdl70sD1o5VUwuSaSFaPEnqAuXSWQbIqM13ZbqJb7XxniazUwxW22yB/cYnD1ZH3lEuHI0ACdU93G16q6mUuOQ+/gkmNGEoI8kPOcyGgA0Awku0og9XlUb5JcB1GSDpP6VpIg18k7/3cA3FYy/g13v6n9r4PAF0JcTCwb/O7+FACeMymEuCRZy3f+u81sn5k9aGb8J1ZCiIuS1Qb/twBcB+AmAEcBfI3d0cx2m9leM9s7M80LOQghusuqgt/dj7t77q3drG8D2BXcd4+7T7j7xNAwr8YihOguqwp+Mxtf8uenAOxfH3eEEN2iE6nv+wA+BGCrmR0G8GUAHzKzm9ASuQ4C+HwnJ3N35AX56L+C2mO/P17QSioHl/oyj2ru8Xk5y7IK5LzIFtf3i+AZbkW1/JjZGD/aFdUhaqu/wTPtjuVcfpuZK38EzUUuK2bGn5ehd/BtpYzImwBgpK5eVuXPcxG0cyvyIOszzDLlUh9rHxfJ307asq1E61s2+N39MyXDD3R8BiHERYl+4SdEoij4hUgUBb8QiaLgFyJRFPxCJEqXC3g6Gnl5Vl+RBdlSBWmTFbx05RZkbYFnxTk5FwDkOWmrFGQCrkLBbM2LCpqGFUjLjf39PJvummne2uzMFl5I9PQQb2vV3yj3v3qOr30eyGiztSDbssmfa5b9VgTFMYsgTbPZ5FJlvgo5D+A+VqicFxXw7By98wuRKAp+IRJFwS9Eoij4hUgUBb8QiaLgFyJRut6rr0lklKiXGZO2on58keYRFUaMtDmWhRcVdawGr6+ZB3JeJShomnHZaFOlvJhl7XUuUT3z3KvUNr+TZxBuemdw+RApLajRiZPDfdSWBU9ZHmTT5UxiCzIqm4EsF2XnIZAq86hYK9Gsw+uU4HEnyvPQO78QiaLgFyJRFPxCJIqCX4hEUfALkShd3e03GKpZ+SmrUWJPhdQ4C88V7LIHteKKaIfVy3dsPQ8SOoLWT6HCEWQt9dd4zb3Tvz5bOn7ohcN0zv7XJ6ntWoxT2+Zx7scQypOFHDxRaDZQMYI8LTSDWo5NptAEyUB5oAREdfVY263W+bjaUmc1GaMWdkT+CNWIC9A7vxCJouAXIlEU/EIkioJfiERR8AuRKAp+IRKlk3ZdOwF8F8B2tHJz9rj7N81sM4AfALgarZZdd7j76ehYmRn6K+WyRjVKZDEik1gglRlPSEFQcy/svEUScaLac5Ug0YLVYQOAapX7P3uM19z75b5DpeOjo8N0znWD76S2G951LbVty7nUtzh5onT81flTdE7WP0htTLIDgEaTr0ejKJffmsFzZkH7ryAXK6z951FmErn2oyQdJlWut9TXBPAld78RwC0AvmBmNwK4B8CT7n49gCfbfwshLhGWDX53P+ruz7VvTwE4AGAHgNsBPNS+20MAPrlRTgoh1p8Vfec3s6sB3AzgGQDb3f1o23QMra8FQohLhI6D38yGATwK4Ivufm6pzVtfNEq/bJjZbjPba2Z7Z6ZJe24hRNfpKPjNrIZW4H/P3X/UHj5uZuNt+ziA0h+Iu/sed59w94mh4fp6+CyEWAeWDX5r1RJ6AMABd//6EtNjAO5s374TwE/W3z0hxEbRSVbfBwB8FsALZvZ8e+xeAPcDeMTM7gJwCMAdyx3IANTI600tC6Q5MieSVjIPHtoqpT6w2m5B6ycLJMxI6osKFBbBa/Y117+7dHzbGK+PZ87bQvn8HLW9vO8Vajt0rDy7cHqQPy/Xj3MfPVJug4KN1az8sdVIdikAVIM2WUVQ36/JpwHBMZmsG7Vlo7UJV8Cywe/uT4OXw/zImj0QQvQE/cJPiERR8AuRKAp+IRJFwS9Eoij4hUiUrhbwhANWlAsHUesqJ25mgRwWFdWM9DzLg8KfRbktC6SyDNxmFkhsgfvVKi8GOb5ta+l4jWRTAsDC3Cy1Hfjta9R28sw0tZ1aLPdxbBPPBOyrBfJbndsiGZBJc1Ehzog852ufE1kRAPn965vOlF9XUVYfyxZdSYsvvfMLkSgKfiESRcEvRKIo+IVIFAW/EImi4BciUbor9YH3M8ubgV5D5ItIzauEhQyDjLmgJ1zOThhJjoEtko2KnEs2W/o2U9vkyfIaqrU6L4450NdPbdt2XE5t/UF9hqGT5Vl9Rc7lwb4qLwbVX+fnip5qZsuCgprN4HlpRL36gusqKvLq7JjBA6uSXDtJfUKIZVHwC5EoCn4hEkXBL0SiKPiFSJSu7va78x1zawQ1yegOZrRjG+16BruyQeunZpMkiUSl+PjGMQYHRqnt8lG+o39F3zi1vbT/v0vHFxvH6Jwb33MztU1M7KK2k4dfoLYjh8rX8diZIPmlyRcyqrkX1V3MKmxelNwV7cxzH5tB+7hI9TFyzGjnPiM+WhATbzlGx/cUQrytUPALkSgKfiESRcEvRKIo+IVIFAW/EImyrNRnZjsBfBetFtwOYI+7f9PM7gPwOQBvtO96r7s/Hh3L3ZGzxJ58gc8j45WgLp0Fr2seaENMzotsUS5FUePHqwwHyUxz/Kk5fOIQtWXV8vU9M3mudBwAXnhxP7UNjF5GbVe971Zq8/qB0vHJX75E50T1E6M15nIeUMnKE4IK55JuKLEFUlolSDDKAokwo7X/gnMRfXkltQk70fmbAL7k7s+Z2QiAZ83sibbtG+7+9x2fTQhx0dBJr76jAI62b0+Z2QEAOzbaMSHExrKi7/xmdjWAmwE80x6628z2mdmDZsY/HwohLjo6Dn4zGwbwKIAvuvs5AN8CcB2Am9D6ZPA1Mm+3me01s70zM4vr4LIQYj3oKPjNrIZW4H/P3X8EAO5+3N1zb+2efRtA6Y/A3X2Pu0+4+8TQEK/GIoToLssGv7W2Ph8AcMDdv75kfGl2yacA8C1jIcRFRye7/R8A8FkAL5jZ8+2xewF8xsxuQkuJOwjg88sdyN3RaJSnueVFkP5GJI9KFqRzBTJJ1AapGWQX5kTq8yBDzIOigMdO/I7aXmsepLYrBvj2ys03XFk6/uroAJ1z/MQctT3z9NPUdvqGP6A21h4sD56XRoPLb5nxOoORrMukLw/aw4WyXCClBUl9gAVyJGnbFknSVSL1raQJWSe7/U+TY4aavhDi4ka/8BMiURT8QiSKgl+IRFHwC5EoCn4hEqXrBTwXiZQWdEiiRC2XovZIkR6SN7k012yWOxnJK6EMRbO5gKzGbV7lxxwaKS8K+o7t/AdWjTle3PPw5HFq+6+fPkFt/ZtGSsf7+vhq1U9yreyKK3hB0wgmv4XZeVFLrqBaa9Q2rOJBBieRHYuC/yJ2Bcl7FL3zC5EoCn4hEkXBL0SiKPiFSBQFvxCJouAXIlG6LvU1Fsv1EFbYM7K5cymkKLjUl4VyTdTHjxXw5LJcpRL4YUEWW5P7MZvxtTp5dr50/Pmf76Vzmgvcx4EqlwirFe7HzOzp0vGi6KNzps7y56UouK1SWUVfxuB5jq6BaF5WcD+qgbRIZekgI5RqfVFm4QXonV+IRFHwC5EoCn4hEkXBL0SiKPiFSBQFvxCJouAXIlEU/EIkioJfiERR8AuRKAp+IRJFwS9Eoiyb2GNm/QCeAtDXvv8P3f3LZnYNgIcBbAHwLIDPepRpg1YuQq1enpCQNbkrtVq5Lcv4nA2p4ZetvIZfrcZfX/v6eV23qIbfYJ3btoyWt7W6edcEnXPoYFDD71Vew6+Zcz+GVlHDb2R0kNqyoDVblIdjrE1W1HYrKpAX1vfj05qRk6wmY8HXl150K6jt18k7/wKAD7v7+9Bqx32bmd0C4KsAvuHu7wJwGsBdnZ9WCNFrlg1+bzHd/rPW/ucAPgzgh+3xhwB8ckM8FEJsCB195zezSrtD7ySAJwD8FsAZd3/zc/BhADs2xkUhxEbQUfC7e+7uNwG4EsAuAO/p9ARmttvM9prZ3tnZcEtACNFFVrTb7+5nAPwMwB8DGLP/3025EsARMmePu0+4+8TgIK8KI4ToLssGv5ltM7Ox9u0BAB8FcACtF4E/a9/tTgA/2SgnhRDrTyc1/MYBPGStQnUZgEfc/d/M7CUAD5vZ3wL4JYAHljuQGVAn0legGoHpF5Wg3VX00DwodNZs8GPmRI50cBmqr4/7UQ/abjUaXHK0Kj/fzNTZ0vHXj5+kc07NzFFbbaRcsgOAD++6hc+rlD/uA/v30TmXbeFSXxO83qEFzzUr71dEtRojW1AbMqz/GFwj7Dr2LKgXuBJNj7Bs8Lv7PgA3l4y/gtb3fyHEJYh+4SdEoij4hUgUBb8QiaLgFyJRFPxCJIqFrYnW+2RmbwA41P5zK4ATXTs5R36cj/w4n0vNj6vcfVsnB+xq8J93YrO97s7zTOWH/JAfG+qHPvYLkSgKfiESpZfBv6eH516K/Dgf+XE+b1s/evadXwjRW/SxX4hE6Unwm9ltZva/Zvaymd3TCx/afhw0sxfM7Hkz29vF8z5oZpNmtn/J2GYze8LMftP+/7Ie+XGfmR1pr8nzZvbxLvix08x+ZmYvmdmLZvYX7fGurkngR1fXxMz6zeznZvarth9faY9fY2bPtOPmB2a2tgIZ7t7VfwAqaJUBuxZAHcCvANzYbT/avhwEsLUH5/0ggPcD2L9k7O8A3NO+fQ+Ar/bIj/sA/GWX12McwPvbt0cA/BrAjd1ek8CPrq4JWjnsw+3bNQDPALgFwCMAPt0e/0cAf76W8/TinX8XgJfd/RVvlfp+GMDtPfCjZ7j7UwBOXTB8O1qFUIEuFUQlfnQddz/q7s+1b0+hVSxmB7q8JoEfXcVbbHjR3F4E/w4Ary35u5fFPx3AT83sWTPb3SMf3mS7ux9t3z4GYHsPfbnbzPa1vxZs+NePpZjZ1WjVj3gGPVyTC/wAurwm3Siam/qG363u/n4AfwrgC2b2wV47BLRe+YGg3NDG8i0A16HVo+EogK9168RmNgzgUQBfdPdzS23dXJMSP7q+Jr6Gormd0ovgPwJg55K/afHPjcbdj7T/nwTwY/S2MtFxMxsHgPb/k71wwt2Pty+8AsC30aU1MbMaWgH3PXf/UXu462tS5kev1qR97hUXze2UXgT/LwBc3965rAP4NIDHuu2EmQ2Z2cibtwF8DMD+eNaG8hhahVCBHhZEfTPY2nwKXVgTa/XHegDAAXf/+hJTV9eE+dHtNela0dxu7WBesJv5cbR2Un8L4K965MO1aCkNvwLwYjf9APB9tD4+NtD67nYXWj0PnwTwGwD/CWBzj/z4ZwAvANiHVvCNd8GPW9H6SL8PwPPtfx/v9poEfnR1TQD8EVpFcfeh9ULz10uu2Z8DeBnAvwLoW8t59As/IRIl9Q0/IZJFwS9Eoij4hUgUBb8QiaLgFyJRFPxCJIqCX4hEUfALkSj/BzJHtTNINnVnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(data.trn_ds.denorm(x)[0]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHGNJREFUeJztnWuMpGd15/+n7n2d7p6bxzNjjy8DCYHYRrMOUUiWTRTioEgGaYXgA/IHlIlWQVqk5INFpECk/UCiAOITqyFYcSLCZQMIa4V2Q6ysLKSVw8Aa2zAxNjCei2d6Lt0909OXup79UOVk3Hn+p6tvVTM8/580murn1PO+p5+3Tr1dz7/OOebuEELkR2HYDgghhoOCX4hMUfALkSkKfiEyRcEvRKYo+IXIFAW/EJmi4BciUxT8QmRKaSuTzewRAJ8FUATwV+7+yXWef1t/nZC9U9omjxfNixZqszbqR+CIBV5aMDGyMTb7bdNOp7NhPwoFft+LbJGP7XZ7w34AQKFY3Pi5WulzdeBw974W3za74GZWBPBjAL8N4ByA7wL4oLv/KJhDT5b+9XvzNuFf9FvxSxT7USPhX4oCBPyFGc3z4DdoBLY6OWQxeEGXNhkI5UqZ20rkvhK8LpuNJrVFrKyuUluJBNbExASdMzI6Sm31wMdr165RWyVYq/Hx8eR4iwQ4AFydm0uOr7ZbaHunr5DZyp/9DwN4xd1/6u4NAF8G8OgWjieEGCBbCf6DAM7e9PO53pgQ4jZgS5/5+8HMjgM4vtPnEUJsjK0E/3kAh2/6+VBv7A24+wkAJ4Dbf8NPiJ8ntvJn/3cBHDWze8ysAuADAJ7aHreEEDvNpu/87t4ys48A+N/obpI/4e4/3DbPhsRmZLT4zxluNYskpeiI3Fgg+7wTY2N0TmQrkt1yACiXK9RWX11Jji+v1ukcD9YjlMpKfCe9RWTARpurMLsnp6ltPFA/pvfuo7ZIs2K/WrMerBUZn52/GvjwRrb0md/dvwXgW1s5hhBiOOgbfkJkioJfiExR8AuRKQp+ITJFwS9Epuz4N/zWspkkne1O7ImOtxnbZrP6IjbtI9GNKkV+qcsFLudFkuPq8jK1tdut5HiUrFILEmqOHn0TtS2tcD9eO//vvncGAJiZ2U3n7J6eorYLs7PUNjPNJcLR4He7dOlycrzZ5ElEhw4dSo7P31ikc9aiO78QmaLgFyJTFPxCZIqCX4hMUfALkSkD3+2/FXJ6bwUfAISOxPXxovfs9EGXl5fojMZKOgkHADrBdn+rld7RB3hdPavW6JxfeNuD1PbAQw9R20rgf6Wa3mUvBtnlK0t8x3z5Wrp8FgA0gnnRbj9bYwtqAraa6bXfSFk+3fmFyBQFvxCZouAXIlMU/EJkioJfiExR8AuRKUNI7Nl4GoxtrgnVJiybPdcmidpkRTXrgnkF4n+bSEMAr3O3LoGPbBUPHTxMLMD9b3oztU0FiTg+P09t9WYjOT5/+RKds3TlIrVFyTYWXJilazzURkdH0scr8RqJ83NpGZAlVKXQnV+ITFHwC5EpCn4hMkXBL0SmKPiFyBQFvxCZsiWpz8xOA1gE0AbQcvdj680pEn3LwOvIbSYPrxBKfTxbqhjIaEUm5QSZVB6paEFmWaCioRz4XyITPTheKzIGjrSCyzI9la5n96ZAzpsO5LxSNS2HAUCjyWW7186eTo4vzV2hc8pBNl21xEOmWuHSXCM45o3r15PjrUgLJjUZo0zLtWyHzv+f3J2vpBDilkR/9guRKVsNfgfwD2b2PTM7vh0OCSEGw1b/7H+nu583s30Avm1m/+Luz9z8hN6bgt4YhLjF2NKd393P9/6/BOAbAB5OPOeEux/rZzNQCDE4Nh38ZjZmZhOvPwbwbgAvbpdjQoidZSt/9u8H8I1e9lkJwN+5+/+KJhi4BNcJc+02LvV5cLxIBiwW+fthG0S3C/S8UpDp1WLHA1AKfIyO2W6TrL5AVoxodfi8UlCM8z4i6R2++widMz4xSW3R6yNSKpv1dCuvTtA2bKTMw+LA/v3UNr6L+79UX+W2pXRx1YUbN+ic5TrPLuyXTQe/u/8UwANb9kAIMRQk9QmRKQp+ITJFwS9Epij4hcgUBb8QmTLQAp6jY2N44G1pgeCll16m8+bnN543ND2dzioDgDe/+Si1tdtcQjl//kxyfHF+gc7xqHBmIDcVy1y/qo7w7LEiyfYa3TVF5+ya5tl05UqV2sYm+THvOHBnes7EOJ0zPbOL2qxQprazP+PXjBUunajx4x3ay+W8WiBvRkVSi0F25OTkRHJ8ZHyMzlmtpwuTrlzkGY5r0Z1fiExR8AuRKQp+ITJFwS9Epij4hciUwe72j47igQd/OWmbD1ouzc9f3vC59u3bQ23MBwBoB62r7rzrUHL83Jm0CgAAl8+dp7alxXTtNgCYGOG7yvcc4S2v7r4rbdt7R3r3HQDGJ/iufbPN1+PGCk9WYdNKZV6rsRAkVZWDZJsb16/xeSSh6dAd/PWxa4QrEo0GVxaqQXutVoMrO6xvW7XEFQlWL7B4qX9lTHd+ITJFwS9Epij4hcgUBb8QmaLgFyJTFPxCZMpApT7vdNAk8tDKNZ4csxkno+MxHwBgIkgIQiH9XjkWSEP3HL6b+1FfobbRKpeNDh88QG0H7kgnpVRHRukcFLj8Nr/A5ciVIJFlbDS9JmMkiQUA6g1+Xer1OrWtLvNad7sn035M1riUurLIj1cKEp0mxvnrYAw8SadN2r15UOOx0Uwn9hSiPm9rn9v3M4UQP1co+IXIFAW/EJmi4BciUxT8QmSKgl+ITFlXRTOzJwD8HoBL7v7W3tgMgK8AOALgNID3uztPy+vRbrawMJvO0FsOsvq4KMOJjsd8AIC7Dx/hflTTcs30rhk6Z2mJS2W1Cl/+yDYV1MEbnUhLaR60PHPj94AxcjwAKBM5DwCqIyPJ8ckpXqfvxnK6tRYAXAmu2UiFZ7+xteo0uXToQQKeBW3D6oH/5UBaZN3XnEiAAGDtKEuwP/q58/81gEfWjD0O4Gl3Pwrg6d7PQojbiHWD392fATC3ZvhRAE/2Hj8J4L3b7JcQYofZ7Gf+/e5+off4Irode4UQtxFb3vDz7gcT+uHEzI6b2UkzO1lvpL+SKIQYPJsN/lkzOwAAvf9ppwB3P+Hux9z9GCs9JIQYPJsN/qcAPNZ7/BiAb26PO0KIQdGP1PclAO8CsMfMzgH4OIBPAviqmX0YwKsA3t/PyRr1Os797HTSVulwWaMQZJ0xSsHxmA8A8Iu/+FZqm55Jt7VqB2+hhRI3Li3ywpN7dnP5cIxkqgFAk0hAS0s8U61AWnwBQCEoSjkVZECOT6YlvUqNH69Dim0CNKESANAiGW4AUPB0ZlwxkOwKVS4dVsrcFmXUtVtB27YOsRHfAQAdIvUF8uBa1g1+d/8gMf1W32cRQtxy6Bt+QmSKgl+ITFHwC5EpCn4hMkXBL0SmDLSAZ6PZwLmzryZtFmSdFVnaU0B0POYDAPzLqR9S2zt+/T8mx8vRl5cKXJabHOOZXmOj3OZMGgJwjcmHxi91p8kzxJorXCIcm+Q9/qqs0GVwKUcCGbBV58U9r125yv0opLMLi87XsFjj0nJthBfwrAZFV4OXI2BkXtC70ElMlF7lvSHXoju/EJmi4BciUxT8QmSKgl+ITFHwC5EpCn4hMmWgUh8AsFqRhRKXV7yz8feoQiAPRoc79dKPqO3AXYeS44ePHOF+BJlZU1OT1FYscG1ofo4Xs1wkPQp333EXnbO8wrPiWm0uiVlQ+LNOpDkr8etSrXIZbf/ePdR212H+uy0U0zLajaDAa7nGfRwZS0uHAFAp8XAqB2mJlVI6U9DLXDp0cq5i4MNadOcXIlMU/EJkioJfiExR8AuRKQp+ITJloLv9ZgYrpnc2y+Vgp3ffHRs+19yli9TW6PAd+CtXrlDbj186lRzfuzdd2w8AipGK0eI7x1bm78vtFk/EaZPfrdFo0jlRjcTR0VFqi5idnU2OV4L6eFPTPFFocprXNDz6tl+itqt79ybHL5w5Q+c0bwSt40b5NSsXgvZrwc49Uwk6Frx2SL3AQqDA/Lvn9v1MIcTPFQp+ITJFwS9Epij4hcgUBb8QmaLgFyJT+mnX9QSA3wNwyd3f2hv7BIDfB/B6hsnH3P1b65/OaPunyd376KyZOw6uf+g1tJy/r81dpX1FUWjyWnGzF15Lj58/S+ccuPNOaus0ufx2fZn7Ecqi+9Nr1QiymRr1OrWNjHBpqxF0Xe4QyXF1hf9e122R2q5d463NUOP1DmeOpJN+LKiROHfmNLUVApk4SnTyoCVaixQ2bLX5uZz40X+zrv7u/H8N4JHE+Gfc/cHevz4CXwhxK7Fu8Lv7MwDmBuCLEGKAbOUz/0fM7Hkze8LMeLtWIcQtyWaD/3MA7gPwIIALAD7Fnmhmx83spJmdbEcth4UQA2VTwe/us+7edvcOgM8DeDh47gl3P+bux4ob+N6xEGJn2VQ0mtmBm358H4AXt8cdIcSg6Efq+xKAdwHYY2bnAHwcwLvM7EF0lYXTAP6gn5NZoYhSLd2+anQ6nX0FAF4e6+fwfR/v+tIytbVaXPa6cjEt9f3f7/wfOud33v271Dazi9fwuzbPpa3KSNDmazItzRVaXARqtYLWVcXNZfzNzaX3iOeu8r3jo/ffT20d5/6Xg9p/VVJzL/oIWgjaoTVXVrhtNaiFGGRVLhOptd3iPnba6cxOltWZYt3gd/cPJoa/0PcZhBC3JPoQLkSmKPiFyBQFvxCZouAXIlMU/EJkymDbdZmhQGSZQiUtAQJAq8CLHzKi4zEfAKC1EMhenXRGWmOJZ6M1V7isiKAVFjpc2mq3ua1AikiOjfPfmfZQAzA5yeXIQtCC6uWXX06O793LszfHJ3ZRmwVZcRb4YaQIZiXIjJzYzWXiSiB9dpr8etaDbMYVIj2vkpZnAFBfTUuOBbXrEkKsh4JfiExR8AuRKQp+ITJFwS9Epij4hciUgUp9xVIJu6ZIX7syz1RbDbKb6LmC41EfADQXea++EumRNxEUg7xxjfd9Y0UYAaBa4fLmlQWe8Vespn3ZNR1IfUHZx1IgHdWDAp733HNPcvzee3nmXjsoWIlOusglAFSDPnhtkrFYNd4zsFPh19OC3ovVGr+XjoxzyXR6T3q83uHrW2+kZcBKLbrOb0R3fiEyRcEvRKYo+IXIFAW/EJmi4BciUwa6218oFDA2PpG0Ndq8xlmT7LJHlINdWeYDALRnAiVgIb37WgL3b36Oqwc3gqSfmf138GOu8nmXL19IjleqfHd7aZkfL6oX6MZ34O9985uS41O7puici+fTvgNAqcJfqqUyt3knfW2qFb4eXuSqw0pQw68RqDflIDGpXEr7UqxyFaNGrqcV+Ot+LbrzC5EpCn4hMkXBL0SmKPiFyBQFvxCZouAXIlP6add1GMDfANiPbgbICXf/rJnNAPgKgCPotux6v7vzLBYA7kCTtBly8DZZ5Q3IF/96vKDtVjOQ5kBq4AFBWb2gvdPSDd6e6uWf/Jjafml6htr+w68co7aL584kx2cvBDJajbfdajS4tLWwuERtlZF0m6yRoMVXocTvRbUalxzb5DUFAJ1yWo4sBtJbrRHU4lvi51paukFtbSLnAQCInFoq8bUqlonUF8iva+nnzt8C8Efu/hYA7wDwh2b2FgCPA3ja3Y8CeLr3sxDiNmHd4Hf3C+7+/d7jRQCnABwE8CiAJ3tPexLAe3fKSSHE9rOhz/xmdgTAQwCeBbDf3V//W/Iiuh8LhBC3CX0Hv5mNA/gagI+6+/Wbbe7uIBUhzOy4mZ00s5OtFi9OIIQYLH0Fv5mV0Q38L7r713vDs2Z2oGc/AOBSaq67n3D3Y+5+rFTaePMNIcTOsG7wW3f78AsATrn7p28yPQXgsd7jxwB8c/vdE0LsFP1k9f0agA8BeMHMnuuNfQzAJwF81cw+DOBVAO9f70BmQJG0VqoENes2Il+8jjuXB9sNLgN2gvfDthObc/86K1wOu3r+HLWdmXqF2vYEmYe1Slpiu/raLJ1z5N77qG0MfB0vzPFagivj15PjxRlSsA7A+PgYtXU/WRIbydzrWdPDwUuqELQvKxW5rRNIjlHGX4lkoBaqwRy+HH2zbvC7+3fAl+q3tu6CEGIY6Bt+QmSKgl+ITFHwC5EpCn4hMkXBL0SmDLSAp5mhSForVYKWS7GUQ84VZAI2AtmoXE1LZQDQrKazrEol7l+Qy4XVhQVqO/MSz/ibGNtFbdO7iQxIJFYAWA2KUkbrYcE6jk+ki6QWg/ZfFrT/ataDb4cGsl2rkS4MG8lyUUZoJDkWi/w1FxX+NHJtSjW+9oVK/2256DG2fAQhxG2Jgl+ITFHwC5EpCn4hMkXBL0SmKPiFyJQBS30FlKvpYoXFIHOv7RuX+lj2IADqAwBMBVlnaKZ72q2s8H58Iy0uDbU6XP5Znr9KbRdPn6a2fXv2Jcf33n2Yzjn32mvUVl1c5LbJSWob25cuQLoaXMtGIPVFRTqbTd7nkUl6jXpQ4LXOexfWV1eprdXihT8387vVRniWY5VkbyKQIteiO78QmaLgFyJTFPxCZIqCX4hMUfALkSkD3e13AJ12ui5ZM9gFbrc3XvK7Q3trAW48AaMStK7avfdAcnxhlvt3eYV3MKsECSn7yvx9uRiUQG8spXfnDwe7/Z1gl/r0yz+jtgfuu5/amu30Mc+eSbcTAwDc4OpHMUjUinb72S57VBbSnK9HJ6jFF/kRqRUs6acSJFXViC1KPFqL7vxCZIqCX4hMUfALkSkKfiEyRcEvRKYo+IXIlHWlPjM7DOBv0G3B7QBOuPtnzewTAH4fwOXeUz/m7t+KjuXuaBBZqRjIK+0Wl1D4uYJEIeNySKDkYIzUVNu1/yCdM3eVv7/Wilz+GR3lNdrqS1w+/MmpF5Ljd7Z4QsoUeP3ECZ7/guWzyd6sAID5+XQrr5XgWtYCOS+SsMpBXcAGkd/KpEUWAASl+FCpbq7Z7GbkyLmrc3ROtZR+fXRa/SfB9aPztwD8kbt/38wmAHzPzL7ds33G3f+y77MJIW4Z+unVdwHAhd7jRTM7BYDf6oQQtwUb+sxvZkcAPATg2d7QR8zseTN7wsymt9k3IcQO0nfwm9k4gK8B+Ki7XwfwOQD3AXgQ3b8MPkXmHTezk2Z2shm0xhZCDJa+gt/MyugG/hfd/esA4O6z7t529w6AzwN4ODXX3U+4+zF3P1behkYDQojtYd3gNzMD8AUAp9z90zeN35zl8j4AL26/e0KInaKf3f5fA/AhAC+Y2XO9sY8B+KCZPYiu/HcawB+sdyB3R7udljysE0h9ZE6EBT2c2oVA6ouUkhbRAY0vY3mCb4V48wa1LQa14pZWg/pzC+msvp+dPkvnTI/yWnwTtXFqu/Qqz9BrEKly6sB+OqcyymsrrtaDjL8ib4o2Wkofs9nka7ga1elr83O121G9RmoCium1qtf5i3FxNW1rbyCrr5/d/u8g3Q0t1PSFELc2+oafEJmi4BciUxT8QmSKgl+ITFHwC5EpAy3gCe8AJLusEWSdoX/14t+OFxT9LBD5BwCKpUDKIe+V7Q5/D6VtlQCUylyOLARXxoICpCuLS8nxG/M8E3A5aCW1bx8/18z4Lmqr1tJrzAq4AkA7KLpaCq7LwsICtZXLaf+bQRHUep1LsEWSTQcAKHCbF/hrrjaRbss1upu366qMTiTHrdh/1qHu/EJkioJfiExR8AuRKQp+ITJFwS9Epij4hciUgUp9pWIRU7vSWWJL13hWHyv6GVEJijqOER8AoFrlvfpQSMtNHXD5ytpcwqwVeLZiJcg8bDT4vDLJLJuM1mMskJR2TVFbYZyvVZn1PAykqKUVfp0rlaBIZ5CJuUokvXaLZ/V5kBHqzu+XDi5HVms8c7I8mpZMC5XgtVgm0mGh//u57vxCZIqCX4hMUfALkSkKfiEyRcEvRKYo+IXIlIFKfcVSEdPT6YKWFlTOnJvnWVuM8Yl01hMATBEfgDhrq0AKRXaCtMPVJS4DVgNZxgIpKiowWSymL2lpjEts7aA53XKTy28jFmQzVtNSVKHIs9sKzl+OHvRXLI/w7EInPSDbQdZn1F+iHYRMocwl5ELkYzU9r13g16zZSq/HBup36s4vRK4o+IXIFAW/EJmi4BciUxT8QmTKurv9ZlYD8AyAau/5f+/uHzezewB8GcBuAN8D8CF351uoPbyT3v0uV3hShBV4ogUjOh7zAQC6fUfTdMi8erA73GryJJyVDt+1by6n224BwOIirzHXcLJWRAUAgKmZPdQ2Mh4k9gT1CdtIKwidNr+WlaBOX4nU4gOAoEsWVuvLyfF6nV/nVpMfsFjlPlYqPEHKg7qRbZLsVCSJZADQJglL273bXwfwm+7+ALrtuB8xs3cA+HMAn3H3+wHMA/hw/6cVQgybdYPfu7x+qyn3/jmA3wTw973xJwG8d0c8FELsCH195jezYq9D7yUA3wbwEwAL/m/foDgH4ODOuCiE2An6Cn53b7v7gwAOAXgYwC/0ewIzO25mJ83sZPTNNCHEYNnQbr+7LwD4JwC/CmDK7F8b0x8CcJ7MOeHux9z9WI00chBCDJ51g9/M9prZVO/xCIDfBnAK3TeB/9x72mMAvrlTTgohtp9+EnsOAHjSuj2iCgC+6u7/08x+BODLZvbfAPw/AF/YQT+FENvMusHv7s8DeCgx/lN0P/8LIW5D9A0/ITJFwS9Epij4hcgUBb8QmaLgFyJTzDeSBrTVk5ldBvBq78c9AK4M7OQc+fFG5Mcbud38uNvd9/ZzwIEG/xtObHbS3Y8N5eTyQ37ID/3ZL0SuKPiFyJRhBv+JIZ77ZuTHG5Efb+Tn1o+hfeYXQgwX/dkvRKYMJfjN7BEze8nMXjGzx4fhQ8+P02b2gpk9Z2YnB3jeJ8zskpm9eNPYjJl928xe7v3Pe4rtrB+fMLPzvTV5zszeMwA/DpvZP5nZj8zsh2b2X3vjA12TwI+BromZ1czsn83sBz0//qw3fo+ZPduLm6+YGe/n1Q/uPtB/AIrolgG7F0AFwA8AvGXQfvR8OQ1gzxDO+xsA3g7gxZvG/gLA473HjwP48yH58QkAfzzg9TgA4O29xxMAfgzgLYNek8CPga4JAAMw3ntcBvAsgHcA+CqAD/TG/zuA/7KV8wzjzv8wgFfc/afeLfX9ZQCPDsGPoeHuzwCYWzP8KLqFUIEBFUQlfgwcd7/g7t/vPV5Et1jMQQx4TQI/Bop32fGiucMI/oMAzt708zCLfzqAfzCz75nZ8SH58Dr73f1C7/FFAPuH6MtHzOz53seCHf/4cTNmdgTd+hHPYohrssYPYMBrMoiiublv+L3T3d8O4HcB/KGZ/cawHQK67/xA0Pd7Z/kcgPvQ7dFwAcCnBnViMxsH8DUAH3X36zfbBrkmCT8Gvia+haK5/TKM4D8P4PBNP9PinzuNu5/v/X8JwDcw3MpEs2Z2AAB6/18ahhPuPtt74XUAfB4DWhMzK6MbcF9096/3hge+Jik/hrUmvXNvuGhuvwwj+L8L4Ghv57IC4AMAnhq0E2Y2ZmYTrz8G8G4AL8azdpSn0C2ECgyxIOrrwdbjfRjAmpiZoVsD8pS7f/om00DXhPkx6DUZWNHcQe1grtnNfA+6O6k/AfAnQ/LhXnSVhh8A+OEg/QDwJXT/fGyi+9ntw+j2PHwawMsA/hHAzJD8+FsALwB4Ht3gOzAAP96J7p/0zwN4rvfvPYNek8CPga4JgF9Gtyju8+i+0fzpTa/ZfwbwCoD/AaC6lfPoG35CZEruG35CZIuCX4hMUfALkSkKfiEyRcEvRKYo+IXIFAW/EJmi4BciU/4/dIpIhnVfjbsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(data.trn_ds.denorm(x)[1]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Initial model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gezi/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/models/cifar10/resnext.py:73: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  init.kaiming_normal(m.weight)\n"
     ]
    }
   ],
   "source": [
    "from fastai.models.cifar10.resnext import resnext29_8_64\n",
    "\n",
    "m = resnext29_8_64()\n",
    "bm = BasicModel(m.cuda(), name='cifar10_rn29_8_64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "data = get_data(8,bs*4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn = ConvLearner(data, bm)\n",
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "lr=1e-2; wd=5e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "lrf = learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(lrf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEOCAYAAABmVAtTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGX6//H3nUIaIUAIPRBK6J3QEXVtuCoqYl8L9o66q7v6c2373bWtumsX+9oVUFFBRERR6T0Qeu8JhIR0Uu7fHzNkszEJk5jJmXK/rutcOXPmzMznISF3znnOeR5RVYwxxhiAEKcDGGOM8R1WFIwxxpSzomCMMaacFQVjjDHlrCgYY4wpZ0XBGGNMOSsKxhhjyllRMMYYU86KgjHGmHJWFIwxxpQLczpAbbVo0UKTkpKcjmGMMX5l2bJlB1U14Xj7+V1RSEpKYunSpU7HMMYYvyIiOzzZz04fGWOMKWdFwRhjTDkrCsYYY8pZUTDGGFPOioIxxphyVhSMMcaUs6JgjDEVHM47yqYDOU7HcIzf3adgjDHeUFRSylu/bOfF7zeTU1TCqT1bcf/ve9A5obHT0RqUFQVjTFBTVWau2c9jM9exK7OAU3q0pG/7OF6bt5XTn53HlSOSmHRKMnHR4U5HbRBWFIwxQW3Kst3cM2U1PVrH8t61wxid3AKAy4Z14JlvN/LW/G3MWX+A968bRvtm0Q6n9T7rUzDGBK3cohKenLWBQR2a8vUdJ5QXBICWsZE8fkE/Pr1xBIfzjnLRKwvYfjDPwbQNw4qCMSbg7D6cz0YPOotf/mEzGTlFPHhOb0JDpMp9UpKa88H1wyksKeOiVxcEfCe0FQVjTEAoLC7li5V7+MPrizjhybmc/fzPrNt3pNr9d2Xm89pP2zh/YDsGJDat8b37tIvj4xuGo8DFkxeyeFtmPaf3HVYUjDF+b/XuLIb9Yw6TPlrJ9kN53P67ZOKiwrnjwxUUFpdW+ZrHv1lPiMC9Y7t79BnJrWL55MYRxESEctGrC7hvWirZBcX12QyfYEXBGOP3Hp+5nvDQED64fhjz7jmZu0/rxtMX9mdTei5//3rdr/Zfsj2Tr1fv48YxXWgTF+Xx53RqEcM3k8Zw3ehOfLxkJ6c8/SNfrtqLqtZncxxlRcEY49cWbT3E/C2HuPmkLozs0oIQd9/AmG4JXH9CJ95duIPZaQfK99+wP4cHv1hL6yaR3Hhi51p/XkxEGA+c3Yvpt42mTVwkt3+4gr9+sYbi0rJ6a5OT7JJUY4xf+/ecTSTERnD5sA6/eu5PZ3Rn/pZD3DtlFX88vTvTlu9m+c4sGoWG8NylA4luVPdfgX3axfHZLSN56tsNvPrjVrYdzOOlywb7/f0MdqRgjPFbx44SbjqxC5Hhob96PiIslOcuHUhhcRkPfL6GrIJiHjirJwvvP4WxfVr/5s8PCw3hvjN78tSEfizelsn5L/3CNj+/bFX87VxYSkqK2nScxhiAy15byKb0XH669+Qqi8Ixq3ZlUVRSxpCkZohUfenpb7V4WyY3vruUEBFm3TWGFo0jvPI5dSUiy1Q15Xj72ZGCMcYvHe8ooaL+iU0Z2qm51woCwNBOzfnohhEcKSzmb1+lee1zvM2KgjHG72TnF/P07I3V9iU4pXvrWG4+qStfrNzLDxvSnY5TJ9bRbIzxC4dyi5iRuo9Zaw+wcOshSsqUv53b+7hHCQ3t1pO78PXqvTzw+Rq+vWvMb+rMdoIdKRhjfF52fjFnP/8zf/1iLXuzCrjuhM58dstIrhiR5HS0X4kIC+Wx8f3YfbiAZ2dvdDpOrflXCTPGBKVHv0ojPaeIj24YzvDO8U7HOa6hnZpz6dAOvPHzNsb2ac2gDt7r4K5vVhSMMT7tu7QDTF2+mzt+19UvCsIxfzmzB3PWHeCClxcQGxFGp4QYkuJjaNM0koTGESTERpDYPJqBiU19qmBYUTDG+Kzs/GLu/yyVHq1jue13yU7HqZW4qHCm3DSSOesPsP1gHlsP5rF852HS1xRxtMLdz3f8rit3n+7Z+EsNwYqCMcZnPfLlWjLzjvLm1UNoFOZ/XaAd4qOZOKrT/2xTVY4UlJCRW8TLP2zhue83065ZFBcP8Y2rqKwoGGN8ypHCYtbsyWb+5kNMW7GHO05Jpk+7OKdj1RsRIS46nLjocB6/oC/pOYXc/9kaWsdFcWK3BKfjWVEwxviG9fuPcOv7y9mS8d9hIkZ2iee2k7s6mMq7wkNDeOnyQVz06kJueW8ZH984wvECaEXBGOMTvl+fzpaMPO4+rRv9E5vSt10czWMaOR3L62Ijw3l74hDOf/EXrn5rCa9eMYjBHZs7lsf/TtIZYwLSpgO5tImL5I5TkjmxW0JQFIRjWjWJ5D/XDiUmIpRLJi/kPwu2OzZHgxUFY4xP2LA/h+RWsU7HcEzXlrFMv200Y5ITePCLtdz9ySoKjlY9a5w3ea0oiEikiCwWkVUislZEHqlh3wtEREXkuCP4GWMCT2mZsiUjl24tGzsdxVFxUeG8dmUKd5/Wjc9X7uHUZ37klR+3cDjvaINl8OaRQhHwO1XtDwwAxorI8Mo7iUgsMAlY5MUsxhgftjMzn6KSMroF8ZHCMSEhwh2nJPP+tcNIbB7F4zPXM/yxOdzz6SrW7s32+ud7raNZXSfEct0Pw91LVSfJ/gY8AdzjrSzGGN+28UAOAMmtgvtIoaKRXVswsmsLNuzP4T8LtjNt+R5aNomgd1vvXp3k1auPRCQUWAZ0BV5U1UWVnh8EJKrq1yJiRcGYILWpvCjYkUJl3VvH8vfz+3Lv2B5V/1ldz7za0ayqpao6AGgPDBWRPseeE5EQ4Bngj8d7HxG5QUSWisjSjIwM7wU2xjhi44Fc2jWNonGEXSVfnbio8AaZ/7lBrj5S1SxgLjC2wuZYoA/wg4hsB4YD06vqbFbVyaqaoqopCQnO3/FnjKlfGw/k2KkjH+HNq48SRKSpez0KOA1Yf+x5Vc1W1RaqmqSqScBCYJyq2gTMxgSRktIytmbk0d1OHfkEbx4ptAHmishqYAkwW1W/EpFHRWScFz/XGONHdmTmc7S0zPoTfIQ3rz5aDQysYvuD1ex/kreyGGN817FO5m52+sgn2B3NxhhHbTzgunK9a5DfuOYrrCgYYxy18UAOic2j/G6C+0BlRcEY46hNB3Lp1tL6E3yFFQVjjGOKS8vYejDXOpl9iBUFY4xjdhzKo7hUrZPZhwRVUSgtc2Z8cmNM1Tbsd3Uy20B4viNoisLcDemc9M+5pB8pdDqKMcZt44EcRKBLgh0p+IqgKQpJ8TEcyC7iHzPWOR3FGOO2KT2HDs2jiWoU6nQU4xY0RaFTixhuOrEzn6/cy4Ith5yOY4zBdY9Csl155FOCpigA3HJyVxKbR/HgF2soLi1zOo4xQe1oSRnbD+bRvbWdOvIlQVUUIsNDefic3mxKz+XNn7c5HceYoPbRkp2UlCkDE5s5HcVUEFRFAeCUnq04tWcr/j1nE/uyC5yOY0xQ2n04nydmrueE5Bac0rOl03FMBUFXFAAeOqcXZar89fO1lNhpJGMalKpy/2drUOAf5/dFRJyOZCoIyqKQ2Dyae87owXfrDnDDu8vIKypxOpIxQWPq8j3M25jBn8f2ILF5tNNxTCVBWRQArh3dib+f34cfN2Zw4SsL2J9t9y8Y423pOYX87as0Ujo244rhHZ2OY6oQtEUB4PJhHXnjqhR2HMrjvBd/Ye3ebKcjGRPQHvkyjYLiUp6Y0I+QEDtt5IuCuigAnNS9JZ/eNBKA8S/N56PFO1G14TCMqW8ZOUXMTN3HxFFJdgezDwv6ogDQq20Tvrx9NEOSmvOXaanc/ckq62cwpp59s3Y/ZQrjB7Z3OoqpgRUFt4TYCN65Zih3ndqNz1fu4ZwXfiZt7xGnYxkTML5atZeuLRvbiKg+zopCBaEhwqRTk3n/2mHkFJZw3ou/MHneFspsdFVjfpP0I4Us3p7JWX3b2CWoPs6KQhVGdm3BrDvHcHKPBP4xYz2Xvb6QPVl2o5sxdTUjdR+qcHa/Nk5HMcdhRaEazWMa8cofBvPkhH6k7s5m7L/msXp3ltOxjPFLX6fuo3urWJthzQ9YUaiBiHBRSiIzJ40hLiqca95eyq7MfKdjGeNX9mcXsmT7YTtK8BNWFDzQIT6atycOobi0jKvfWkx2frHTkYzxG1+n7gPgLCsKfsGKgoe6toxl8hWD2ZVZwPXvLqWopNTpSMb4ha9X76VXmyZ0tnsT/IIVhVoY1jmepy7sx+Jtmfzxk1U257Mxx7Enq4DlO7PsKMGPhDkdwN+cO6Ad+7MLeWzmeqLCQ3niArtd35jqzFjtOnVk/Qn+w4pCHdx4Yhfyj5by7zmbCAsN4e/n9bHCYEwVZqcdoFebJnSMj3E6ivGQFYU6uvPUZIpLy3jphy2EhQiPntvbbsoxpoKcwmKW7zzM9WM6Ox3F1IIVhToSEe45ozslZcrkeVuJjgjlvjN7Oh3LGJ8xf8shSsqUMckJTkcxtXDcoiAiMUCBqpaJSDegBzBTVYP+ukwR4b4ze5BXVMKrP26lb7s4zu7X1ulYxviEeRsziGkUyuCONgezP/Hk6qN5QKSItAO+Ba4A3vZmKH8iIjx0Tm8GdWjKvVNWszk9x+lIxjhOVZm3KYMRXVrQKMwucvQnnny3RFXzgfHAS6p6IdD7uC8SiRSRxSKySkTWisgjVexzt4ikichqEZkjIn45FVOjsBBeunww0Y1CufHdZeTasNsmyG0/lM+uzAJO7NbC6SimljwqCiIyArgc+Nq9LdSD1xUBv1PV/sAAYKyIDK+0zwogRVX7AVOAJz2L7Xtax0Xy/KWD2H4on3unrLKJekxQm7cxA4Ax3aw/wd94UhTuBO4DPlPVtSLSGZh7vBepS677Ybh70Ur7zHUfhQAsBPx69o0RXeK594zuzEjdzxs/b3M6jjGOmbcxg47x0XYpqh86blFQ1R9VdZyqPiEiIcBBVb3DkzcXkVARWQmkA7NVdVENu18LzPQotQ+7YUxnxvZuzWMz17Nke6bTcYxpcEdLyliw9ZBddeSnjlsUROQDEWnivgppDZAmIvd48uaqWqqqA3AdAQwVkT7VfMYfgBTgqWqev0FElorI0oyMDE8+2jEiwpMX9iOxWRS3vr+cjJwipyMZ06CW7sgk/2ipnTryU56cPuqlqkeA83D9Jd8J1xVIHlPVLFynnMZWfk5ETgX+HzBOVav8Daqqk1U1RVVTEhJ8/wetSWQ4L/9hMEcKi7n9w+WUlJY5HcmYBjNv40HCQoQRXeKdjmLqwJOiEC4i4biKwnT3/QnH7UUVkQQRaepejwJOA9ZX2mcg8CqugpBe2/C+rGebJvzfeX1ZuDWTp2dvdDqOMQ1m3sYMBndsRuMIuzfWH3lSFF4FtgMxwDz3ZaOezGjfBpgrIquBJbj6FL4SkUdFZJx7n6eAxsCnIrJSRKbXugU+bMLg9lw6NJGXf9jCT5t8+7SXMfUhI6eItH1H7NSRH5O6XDopImGq6sjF+CkpKbp06VInPrpOCotL+f1zP1FUXMasu8bYX08moH26dBf3TFnNV7ePpk+7OKfjmApEZJmqphxvP086muNE5JljHb0i8jSuowbjgcjwUJ6a0I+92QU8MXP98V9gjB/7YuVeEptH0bttE6ejmDry5PTRm0AOcJF7OQK85c1QgWZwx+ZMHNmJdxfuYOHWQ07HMcYr9mUX8MuWg4wf2N5GDPZjnhSFLqr6kKpudS+PADYWbi3dc0Z3OsZH8+epqyk4alN5msDz+Yq9qML4Qe2cjmJ+A0+KQoGIjD72QERGAQXeixSYohqF8vj4fuw4lM9TszY4HceYeqWqTFu+m5SOzewuZj/nSVG4GXhRRLaLyA7gBeAm78YKTCO6xHPF8I68NX+b3e1sAsqaPUfYlJ7L+EF+PVKNwbNhLla6B7XrB/RV1YGqusr70QLTX87sQftmUfzxk1Xk2WiqJkBMXb6bRmEhnNXX5mL2d9VeHykid1ezHQBVfcZLmQJaTEQY/5zQn0teW8jjM9fzt/OqHPnDGL9RXFrG9FV7Oa1nK+Kiw52OY36jmo4UYo+zmDoa1jmea0e5rkaym9qMv/txQwaZeUetgzlAVHuk4L7KyHjJn87oztwN6dw7ZTWz7hpDk0j7C8v4p2krdhMf08juYg4QNk+eQyLDQ3n6ogGk5xTxyPQ0p+MYUyfZBcV8l5bOuAFtCQ+1XyeBwL6LDhqQ2JRbTurC1OW7mbV2v9NxjKm15TsOc7S0jNN7tXY6iqknVhQcdvvvkundtgn3T0vlYK7NvWD8y4qdhwkR6J9o4xwFCk/GPooQkctE5H4RefDY0hDhgkGjsBCevXgAOUUl3Dct1eZ2Nn5lxa4serRuQnQjG+gxUHhypPAFcC5QAuRVWEw96dYqlnvP6M7stAN8umy303GM8UhZmbJyZxYDOzR1OoqpR56U9/aq+qsZ00z9umZUJ2anHeDRL9MY0TmexObRTkcypkZbMnLJKSphYIdmTkcx9ciTI4X5ItLX60mCXEiI8M8L+6OqPDR9rdNxjDmuFTuzAOxIIcB4UhRGA8tEZIOIrBaRVPdsaqaeJTaPZtKpyXy/Pp3v1x9wOo4xNVqx6zBxUeF0sgHwAoonp4/O9HoKU+7qkZ34aMkuHv0yjVFdWxARFup0JGOqtGJnFgMSmxISYnMnBBJPBsTbATQFznEvTd3bjBc0Cgvh4XN6s/1QPm/8vM3pOMZUKbeohI0HcuzUUQDy5JLUScD7QEv38p6I3O7tYMFsTLcETuvVihe+38z+7EKn4xjzK6t3Z1GmWCdzAPKkT+FaYJiqPqiqDwLDgeu9G8v89axelJQp/5ixzukoxvzKsU7mAe3tSCHQeFIUBKg4f2Spe5vxog7x0dw0pjPTV+21eZ2Nz1mxM4vOCTE2VHYA8qQovAUsEpGHReRhYCHwhldTGQBuPqkr7ZtF8cDnazhaUuZ0HGMA19SbK3cdZmCinToKRJ50ND8DTAQy3ctEVf2Xt4MZ17zOj4zrzeb0XF7/eavTcYwBYPfhAg7mHrVO5gBVbVEQkSbur82B7cB77mWHe5tpAKf0bMXpvVrx3JxN7MrMdzqOMSzfeRiwm9YCVU1HCh+4vy4DllZYjj02DeShcb0RhEe+tDudjfNW7MwiKjyU7q1sAsZAVG1RUNWz3V87qWrnCksnVe3ccBFNu6ZR3HlqMt+tS+dbm3fBOGzFriz6tY8jzCbVCUie3Kcwx5NtxruuGd2J7q1ieXj6WnIKi52OY4JUwdFS1u7JZlBH62QOVDX1KUS6+w5aiEgzEWnuXpIAm6G7gYWHhvCP8X3Zf6SQR7+06TuNM1bsOkxJmTI0yboVA1VNRwo34uo/6OH+emz5AnjB+9FMZYM7NuPmk7rw6TKbvtM4Y/G2TERgcJIdKQSqmvoU/q2qnYA/VehL6KSq/VXVioJDJp3Sjd5tm3DftFQycmz6TtOwFm/LpFebJjSJtJvWApUn9yk8LyJ9ROQiEbny2HK817lPPy0WkVUislZEHqlinwgR+VhENovIIvepKVODRmEh/OviAeQWlXDftNU2fadpMEdLyli+8zBD7NRRQPOko/kh4Hn3cjLwJDDOg/cuAn6nqv2BAcBYERleaZ9rgcOq2hV4FniiFtmDVnKrWP48tgffrUvn4yW7nI5jgsSavdkUFpcxrJMVhUDmyTVlE4BTgP2qOhHoD8Qd70Xqkut+GO5eKv9Zey7wjnt9CnCKiNi4Sh6YODKJEZ3j+b+v17E3q8DpOCYILN6WCcAQKwoBzZOiUKCqZUCJ+y7ndCDRkzcXkVARWel+zWxVXVRpl3bALgBVLQGygXhPwwezkBDhyQn9KC1T7v8s1U4jGa9bsi2TLgkxtGgc4XQU40WeFIWlItIUeA3X1UfLgQWevLmqlqrqAKA9MFRE+tQlpIjcICJLRWRpRkZGXd4iICU2j+besd35YUMGn63Y43QcE8BKy5TF2zMZakcJAc+TjuZbVDVLVV8BTgOucp9G8piqZgFzgbGVntqD+6hDRMJwnZb61TjRqjpZVVNUNSUhIaE2Hx3wrhqRxOCOzXjkyzTSc2xCHuMdG/bnkFNYYkUhCNR089qgygvQHAhzr9dIRBLcRxiISBSugrK+0m7Tgavc6xOA79XOg9RKSIjwxAX9KCgu5aEvbGwk4x1Ltrv6E4Z2srO7gS6shueedn+NBFKAVbgm1+mHa0C8Ecd57zbAOyISiqv4fKKqX4nIo8BSVZ2Oa16Gd0VkM65huS+pc0uCWNeWjZl0SjJPzdrA16v3cVa/Nk5HMgFm8bZM2jWNol3TKKejGC+rtiio6skAIjINGKSqqe7HfYCHj/fGqroaGFjF9gcrrBcCF9Y6tfmVG8d05tu0A/xl2mr6toujQ3y005FMgFBVFm3L5ITkFk5HMQ3Ak47m7scKAoCqrgF6ei+SqYuw0BBeuHQgAtzywTIKi0uP+xpjPLH9UD4Hc4usPyFIeFIUVovI6yJyknt5DVjt7WCm9hKbR/P0RQNYs+cIf/96ndNxTIBYvM117YfdyRwcPCkKE4G1wCT3kubeZnzQab1accOYzry7cAfTV+11Oo4JAPO3HCI+phFdEmKcjmIaQE0dzUD5ef9n3YvxA/ec0Z1lOw5z39TVdGvVmB6tmzgdyfipvKISZqcdYFz/tthgA8GhpktSP3F/TRWR1ZWXhotoais8NITnLx1ITEQYV7yxmO0H85yOZPzUjNR95B8t5cKU9k5HMQ2kptNHk9xfzwbOqWIxPqxt0yjev24YJaVlXP76IhsfydTJlGW76dQihkEdbP6EYFHTfAr73F93VLU0XERTV8mtYvnPNcM4UlDMH15fxMFcm3/BeG7noXwWbctkwuD2duooiNR0+ihHRI5UseSIyJGGDGnqrm/7ON6cOIS92QVc8cZisgtsfmfjmanLdyMC4wfZ7LvBpKYjhVhVbVLFEquq1nPpR4YkNWfyFSlsTs/h+neW2j0M5rjKypQpy3YzumsL2sTZXczBxJNLUgEQkZYi0uHY4s1Qpv6N6ZbAMxcNYMmOTG7/cAUlpWVORzI+bOG2Q+zJKmDCYOtgDjaezLw2TkQ2AduAH4HtwEwv5zJecE7/tjx8Tm9mpx3g/322xuZgMNWasmw3sRFhnNG7tdNRTAM77n0KwN+A4cB3qjpQRE4G/uDdWMZbrhqZxKHcIp77fjPNGzfiz2N7OB3J+JjcohJmpu7nvIHtiAwPdTqOaWCeFIViVT0kIiEiEqKqc0XkX15PZrzmrtO6cSjvKC//sIXGEWHcenJXpyMZH/Lt2v0UFJcyYbB1MAcjT4pClog0BuYB74tIOmB3Q/kxEeFv5/Yhr6iEp2ZtIKZRKFeP6uR0LOMjFm/LJC4q3O5NCFKeFIVzgQLgLuByXLOjPerNUMb7QkKEf17Yn/yjpTz8ZRrREWFclOLR1NsmwC3feZiBHZravQlBypOrj24E2qhqiaq+o6rPqeqvpsw0/icsNITnLxvICckt+MvU1fz7u00czjvqdCzjoCOFxWxKz2Vgoh0lBCtPikIs8K2I/CQit4lIK2+HMg0nIiyUyVekcErPVjz73UaGPzaH+6atZuOBHKejGQes3pWNKgzq2NTpKMYhxy0KqvqIqvYGbsU1xeaPIvKd15OZBhPVKJTXrkxh1p1jGD+oHdOW7+H0Z+fx1Kz1lJXZZavBZMXOw4hA/0QrCsHK45vXgHRgP3AIaOmdOMZJ3VvH8tj4fiy47xQuGZLIi3O3cMO7S8kptKExgsWKXVl0TWhMk8hwp6MYh3hy89otIvIDMAeIB65X1X7eDmac0zymEY+N78uj5/Zm7oYMxr8034bfDgKqygp3J7MJXp4cKSQCd6pqb1V9WFXTvB3KOE9EuHJEEu9eM5SM3CLGvfAzHy/ZaXdBB7Adh/I5nF/MQLsUNah50qdwn6qubIgwxveM7NqC6beOpkebJvx5aioXT17I5vRcp2MZL1ix6zCAHSkEudr0KZgg1SE+mo+uH84TF/Rlw/4cfv/vn3huziYbVC/ArNiZRUyjUJJbxjodxTjIioLxSEiIcPGQDsz544mM7dOaZ2Zv5OLJC9mVme90NFNPlu88TP/EpoSG2E1rwcyKgqmVFo0jeO7Sgfz7kgFsdB81TF+11+lY5jcqOFrKun05durIWFEwdXPugHbMmHQCya0ac8eHK7jn01UUHLXJe/xV6p5sSsvU7mQ2VhRM3SU2j+aTG0dw28ldmbJ8N+e9+It1QvupFTtdncwD7Egh6FlRML9JWGgIfzqjO+9M/O+lq1+s3ON0LFNLK3Zm0aF5NC0aRzgdxTjMioKpF2O6JTDjjhPo3bYJkz5aycPT11JsVyf5BVVl+c7DDLKjBIMVBVOPWsdF8uH1w7ludCfenr+dq95cbKOu+oF92YWk5xTZTWsGsKJg6llYaAgPnN2Lpy/sz9Idhxn34s+s33/E6VimBhv2u0bE7dW2icNJjC/wWlEQkUQRmSsiaSKyVkQmVbFPnIh8KSKr3PtM9FYe07AuGNyej28YTlFxGRe8NJ9lOw47HclU49jFAV0TGjucxPgCbx4plAB/VNVewHDgVhHpVWmfW4E0Ve0PnAQ8LSKNvJjJNKCBHZox/bbRJMRGMPGtxaTttSMGX7QlI5f4mEY0i7H/esaLRUFV96nqcvd6DrAOqDwTuAKx4pr3rzGQiauYmADROi6S964bRkxEGFe+uYitGXbJqq/ZnJ5LFztKMG4N0qcgIknAQGBRpadeAHoCe4FUYJKq2iUrAaZ9s2jeu24YqvCH1xexJ6vA6UjGTVXZnJFLl5ZWFIyL14uCiDQGpuIafrvy+YMzgJVAW2AA8IKI/Kq3S0RuEJGlIrI0IyPD25GNF3RJaMx/rh1KTlEJF72ywPoYfERm3lGy8ovpakXBuHm1KIhIOK6C8L6qTqtil4nANHXZDGwDelTeSVUnq2qKqqYkJCR4M7Lxot5t43j/umGIwEWvLuD5OZsotek+HXWsk7lLQozDSYyv8ObVRwK8AaxT1Weq2W0ncIp7/1ZAd2CrtzIZ5/Vr35QZk06v3+JkAAASTElEQVTg933b8PTsjVz62kL22ukkx2x29/HYkYI5xptHCqOAK4DfichK9/J7EblJRG5y7/M3YKSIpOKa7vPPqnrQi5mMD2gSGc5zlwzgnxf2Z82ebM741zymLttts7o5YEt6HlHhobSNi3I6ivERYd56Y1X9GahxYHZV3Quc7q0MxneJCBMGt2dIUjP+9Okq/vjpKr5Zu59/nN+XhFgbf6ehbM7IpXNCDCE2h4JxszuajaM6xsfw0Q0jeOCsnvy4MYPTn/2ReRvtYoKGsiU9104dmf9hRcE4LjREuO6Ezsy4YzStmkQy8e0lfLh4p9OxAl7+0RL2ZBXYPQrmf1hRMD6ja8tYPr1pBKO7tuC+aak88c16yuzqJK/ZmpEHWCez+V9WFIxPiY0M542rUrhsWAde/mELt3+0grwiu8ndG8rHPLKiYCqwomB8TlhoCH8/rw/3ndmDGan7OOu5n1i1K8vpWAFnS0YuIQId46OdjmJ8iBUF45NEhBtP7MKH1w/naEkZF7w8nxfnbrab3erR5vRcOsbHEBEW6nQU40OsKBifNrxzPDMnjeGMPq15atYGrnxzEdkFxU7HCgg2EJ6pihUF4/PiosN54dKBPHlBPxZvy+TiVxdw4Eih07H8WklpGdsP5dGlpQ1vYf6XFQXjF0SEi4Yk8ubVQ9iVmc/4l+azxYbhrrOdmfkUl6pNrGN+xYqC8SsnJCfw0Q0jKCwuZcLL81mx00ZbrYstdjmqqYYVBeN3+raPY+rNI4mNDOeSyQuZkbrP6Uh+p3x0VCsKphIrCsYvJbWI4bNbRtKnXRy3vL+cF+dutgH1amFzei4tYyNoEhnudBTjY6woGL8V3ziC968bxrkD2vLUrA386dPVHC2xifs8sSXDxjwyVbOiYPxaZHgo/7p4AHeemszU5bu548MVlJRaYahJUUkpmw7kkGxFwVTBioLxeyLCnad246FzevHN2v38eWqqjZlUg583HSTvaCkn92jpdBTjg7w2n4IxDW3iqE4cKSjh2e82EhsZxkPn9MI1AaCp6OvV+4iLCmdU1xZORzE+yIqCCSh3nNKVI4XFvPHzNppEhnH36d2djuRTCotLmZ12gDP7tiY81E4UmF+zomACiojwwFk9yS0s4bnvN3M4v5gHz+llvwDdftp0kJyiEs7q19bpKMZHWVEwAUdE+Mf4vjSNCefVH7ey4UAOL18+iPjGNs3n16v30jQ6nJFd4p2OYnyU/flkAlJoiHDfmT359yUDWLUri3Ev/MLavdlOx3JUYXEp361LZ2xvO3Vkqmc/GSagnTugHVNuGkmZKhe+siCo53+etzGD3KISzurXxukoxodZUTABr2/7OL64dRQd42O45u0lfLFyj9ORHPF16j6aRYczorOdOjLVs6JggkLLJpF8fONwBndsxqSPVvLmz9ucjtSgCotL+S7tAGP7tCbMTh2ZGthPhwkaTSLDeeeaoYzt3ZpHv0rj0S/TgmZYjB82ZJB3tJSz+tpVR6ZmVhRMUIkMD+XFywdx9cgk3vxlG+Nf/iUo5mX4avVemsc0Ynjn5k5HMT7OioIJOqEhwsPjejP5isHsOVzA2c/9zMdLdgbsKKs5hcXMTjvAWX3b2Kkjc1z2E2KC1um9W/PNnWMY1LEpf56ayjVvL2FXZr7TserdzDX7KSop4/xB7ZyOYvyAFQUT1Fo1ieTda4bx17N7sWhbJqc/O4/J87ZQHEAjrX6+Yg9J8dEMTGzqdBTjB6womKAXEiJcO7oTs+8+kVFd4/nHjPWMe+EXth3Mczrab7Yvu4AFWw9x3sB2Njig8YgVBWPc2jWN4rUrU3jlD4PZn13A+S/9wqKth5yO9Zt8vmIvqnD+QDt1ZDxjRcGYCkSEsX1a8/mto2ge04g/vLGIqct2Ox2rTlSVz1bsZnDHZnSMj3E6jvETVhSMqULH+Bg+u3kUQ5Ka88dPV/HkN+v9bka3tXuPsPFArh0lmFrxWlEQkUQRmSsiaSKyVkQmVbPfSSKy0r3Pj97KY0xtxUW7bna7dGgHXvphCxNeWeBX/QyfrdhDeKhwVl8b68h4zptHCiXAH1W1FzAcuFVEelXcQUSaAi8B41S1N3ChF/MYU2vhoSE8Nr4vz186kG0H8/j9v3/ig0W+f09DSWkZ01ft5eTuLWkW08jpOMaPeK0oqOo+VV3uXs8B1gGVj2MvA6ap6k73funeymPMb3FO/7bMunMMgzs24/7PUrnpvWXkFpU4HataP27MICOniPF2b4KppQbpUxCRJGAgsKjSU92AZiLyg4gsE5Erq3n9DSKyVESWZmQE79DHxlmt4yL5zzVDeeCsnny3Lp0LXprvkze7bU7P4U+frqJD82hO7tHS6TjGz3i9KIhIY2AqcKeqHqn0dBgwGDgLOAP4q4h0q/weqjpZVVNUNSUhIcHbkY2pVkiIcN0JnXln4lD2ZRcw7oWfWbDFdy5b3ZNVwBVvLCY0JIT/XDOUiLBQpyMZP+PVoiAi4bgKwvuqOq2KXXYDs1Q1T1UPAvOA/t7MZEx9GJ3cgi9uG03zmEZc8cYi3vplm+P9DAdzi7ji9UXkFpXw7rVDSWphl6Ga2vPm1UcCvAGsU9VnqtntC2C0iISJSDQwDFffgzE+r1OLGD67dRQndW/JI1+mcdN7y8guKHYky67MfK5+azF7swt46+oh9GzTxJEcxv+FefG9RwFXAKkistK97X6gA4CqvqKq60TkG2A1UAa8rqprvJjJmHrVJDKc164czBs/b+Pxmes567mfeOGyQQxooHGGsvOLeWHuJt6Zv4OQEHj58sGkJNnw2KbuxOlD3tpKSUnRpUuXOh3DmF9ZsfMwt32wgv1HChmY2JQTkhMYndyC/u3j6m3I6rIyZU9WAZszclm9K5u35m8ju6CYCwe35+7TutM6LrJePscEHhFZpqopx93PioIx9Sc7v5jXf97KjxszSN2TjSrERYVzeq9W/L5fG0Z1aUGjMFeBKCwuJSOniCZR4cRFhf/qfVL3ZLN+/xF2Hy5wL/nsOJRPQXFp+X4nJLfgvjN70qutnS4yNbOiYIzDDucd5ZctB5mzLp3v0g6QU1RCXFQ47ZpGsf9IIZl5R8v3bRYdTof4GBIaR7A5PYfth/57qWtMo1ASm0fTrmkUHeKjSW4ZS3KrxnRNaGw3phmPWVEwxocUlZTy08aDzFizj6z8YlrHRdI2LpKWsZFkFRxlxyHXUcCBI4V0ToihX/um9G/flF5tm9AsOtyGvTa/madFwZsdzcYYt4iwUE7t1YpTe7VyOooxNbJRUo0xxpSzomCMMaacFQVjjDHlrCgYY4wpZ0XBGGNMOSsKxhhjyllRMMYYU86KgjHGmHJ+d0eziGQAO6p5Og7IruHl1T1f1fbK2yo+rm69BXCwhs+vjeO1xdP96qvNlR97o92ettmTfT1pX1XbAq3N1W33tJ0VHzvx8328fe3nu+btFR93VNXjz1KmqgGzAJPr8nxV2ytvq/i4hvWlDdWWhm5zQ7Tb0zbXtd3B2Oa6tLu655z4+T7evvbzXbs2erIE2umjL+v4fFXbK2/70oP1+uTp+zZUmys/9ka7a/OedWl3MLa5uu21aacvt9t+vmveXut2+N3pI18mIkvVgwGnAk0wttvaHDyCrd2BdqTgtMlOB3BIMLbb2hw8gqrddqRgjDGmnB0pGGOMKWdFwRhjTDkrCsYYY8pZUWhAIhIjIktF5GynszQEEekpIq+IyBQRudnpPA1FRM4TkddE5GMROd3pPA1BRDqLyBsiMsXpLN7k/j/8jvv7e7nTebzBioIHRORNEUkXkTWVto8VkQ0isllE/uLBW/0Z+MQ7KetXfbRZVdep6k3ARcAob+atL/XU7s9V9XrgJuBib+atD/XU5q2qeq13k3pHLds/Hpji/v6Oa/CwDcCuPvKAiIwBcoH/qGof97ZQYCNwGrAbWAJcCoQCj1V6i2uA/kA8EAkcVNWvGiZ93dRHm1U1XUTGATcD76rqBw2Vv67qq93u1z0NvK+qyxsofp3Uc5unqOqEhspeH2rZ/nOBmaq6UkQ+UNXLHIrtNWFOB/AHqjpPRJIqbR4KbFbVrQAi8hFwrqo+Bvzq9JCInATEAL2AAhGZoapl3sz9W9RHm93vMx2YLiJfAz5fFOrpey3A47h+efh0QYD6+177q9q0H1eBaA+sJEDPtFhRqLt2wK4Kj3cDw6rbWVX/H4CIXI3rSMFnC0INatVmdyEcD0QAM7yazLtq1W7gduBUIE5EuqrqK94M5yW1/V7HA38HBorIfe7i4c+qa/9zwAsichbeG+LGUVYUGpiqvu10hoaiqj8APzgco8Gp6nO4fnkEDVU9hKsPJaCpah4w0ekc3hSQhz8NZA+QWOFxe/e2QBaMbYbgbHcwtrmioG2/FYW6WwIki0gnEWkEXAJMdziTtwVjmyE42x2Mba4oaNtvRcEDIvIhsADoLiK7ReRaVS0BbgNmAeuAT1R1rZM561MwthmCs93B2OaKgr39ldklqcYYY8rZkYIxxphyVhSMMcaUs6JgjDGmnBUFY4wx5awoGGOMKWdFwRhjTDkrCsbrRCS3AT5jnIfDl9fnZ54kIiPr8LqBIvKGe/1qEXmh/tPVnogkVR4+uop9EkTkm4bKZBqeFQXjN9zDGVdJVaer6uNe+Myaxgc7Cah1UQDux0/HRlLVDGCfiPjF/Bim9qwomAYlIveIyBIRWS0ij1TY/rmILBORtSJyQ4XtuSLytIisAkaIyHYReURElotIqoj0cO9X/he3iLwtIs+JyHwR2SoiE9zbQ0TkJRFZLyKzRWTGsecqZfxBRP4lIkuBSSJyjogsEpEVIvKdiLRyD7V8E3CXiKwUkRPcf0VPdbdvSVW/OEUkFuinqquqeC5JRL53/9vMEZEO7u1dRGShu73/V9WRl7hmBPtaRFaJyBoRudi9fYj732GViCwWkVj35/zk/jdcXtXRjoiEishTFb5XN1Z4+nMgIGcdM4Cq2mKLVxcg1/31dGAyILj+IPkKGON+rrn7axSwBoh3P1bgogrvtR243b1+C/C6e/1q4AX3+tvAp+7P6IVrXHyACbiG8A4BWgOHgQlV5P0BeKnC42b89+7/64Cn3esPA3+qsN8HwGj3egdgXRXvfTIwtcLjirm/BK5yr18DfO5e/wq41L1+07F/z0rvewHwWoXHcUAjYCswxL2tCa6RkaOBSPe2ZGCpez0JWONevwF4wL0eASwFOrkftwNSnf65ssU7iw2dbRrS6e5lhftxY1y/lOYBd4jI+e7tie7th4BSYGql95nm/roM13wNVflcXXNWpIlIK/e20cCn7u37RWRuDVk/rrDeHvhYRNrg+kW7rZrXnAr0EpFjj5uISGNVrfiXfRsgo5rXj6jQnneBJytsP8+9/gHwzypemwo8LSJPAF+p6k8i0hfYp6pLAFT1CLiOKnDNCTAA179vtyre73SgX4UjqThc35NtQDrQtpo2GD9nRcE0JAEeU9VX/2ejazKeU4ERqpovIj/gmrYUoFBVSyu9T5H7aynV/wwXVViXavapSV6F9eeBZ1R1ujvrw9W8JgQYrqqFNbxvAf9tW71R1Y0iMgj4PfB/IjIH+Kya3e8CDuCaIjYEqCqv4Doim1XFc5G42mECkPUpmIY0C7hGRBoDiEg7EWmJ66/Qw+6C0AMY7qXP/wW4wN230ApXR7En4vjvWPpXVdieA8RWePwtrlnXAHD/JV7ZOqBrNZ8zH9cQzeA6Z/+Te30hrtNDVHj+f4hIWyBfVd8DngIGARuANiIyxL1PrLvjPA7XEUQZcAWueZcrmwXcLCLh7td2cx9hgOvIosarlIz/sqJgGoyqfovr9McCEUkFpuD6pfoNECYi63DNbbzQSxGm4ppWMQ14D1gOZHvwuoeBT0VkGXCwwvYvgfOPdTQDdwAp7o7ZNKqYiUxV1+OapjO28nO4CspEEVmN65f1JPf2O4G73du7VpO5L7BYRFYCDwH/p6pHgYuB590d9bNx/ZX/EnCVe1sP/veo6JjXcf07LXdfpvoq/z0qOxn4uorXmABgQ2eboHLsHL+45hReDIxS1f0NnOEuIEdVX/dw/2igQFVVRC7B1el8rldD1pxnHnCuqh52KoPxHutTMMHmKxFpiqvD+G8NXRDcXgYurMX+g3F1DAuQhevKJEeISAKu/hUrCAHKjhSMMcaUsz4FY4wx5awoGGOMKWdFwRhjTDkrCsYYY8pZUTDGGFPOioIxxphy/x/JvS3JZbK4AAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2989b096d1d9428b844d90c0e1adc6e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 34/98 [00:14<00:27,  2.33it/s, loss=2.23]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<timed eval>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/learner.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lrs, n_cycle, wds, **kwargs)\u001b[0m\n\u001b[1;32m    302\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cycle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarm_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    249\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 251\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    252\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, visualize, **kwargs)\u001b[0m\n\u001b[1;32m    139\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_stepper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m             \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mavg_mom\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mavg_mom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m             \u001b[0mdebias_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mavg_mom\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/model.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, xs, y, epoch)\u001b[0m\n\u001b[1;32m     72\u001b[0m             \u001b[0mcopy_fp32_to_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp32_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msynchronize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/model.py\u001b[0m in \u001b[0;36mtorch_item\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mtorch_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'item'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mStepper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "%time learn.fit(lr, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb461e4ecfe447c5842e9783ba6012cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 9/98 [00:04<00:44,  2.01it/s, loss=2.18]\n",
      "epoch      trn_loss   val_loss   accuracy                 \n",
      "    0      1.660695   1.51586    0.4534    \n",
      "    1      1.511793   1.423011   0.4882                   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.4230109336853027, 0.488200000667572]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 2, cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6233cac5f3244aa9af4ca94b9b307c59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                 \n",
      "    0      1.417075   1.3854     0.5045    \n",
      "    1      1.396977   1.360235   0.5086                   \n",
      "    2      1.262032   1.29839    0.5347                   \n",
      "    3      1.323917   1.434224   0.4868                   \n",
      "    4      1.226122   1.269455   0.5412                   \n",
      "    5      1.106373   1.196596   0.5697                   \n",
      "    6      1.019568   1.186597   0.5787                   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.186597248840332, 0.5787000005722046]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 3, cycle_len=1, cycle_mult=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.save('8x8_8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## 16x16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.load('8x8_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.set_data(get_data(16,bs*2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81c03dc817974984bd24b498e9621185",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.517224   1.479416   0.4629    \n",
      "\n",
      "CPU times: user 25.1 s, sys: 22.8 s, total: 47.9 s\n",
      "Wall time: 26.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.4794159097671509, 0.4629]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-3, 1, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "506863548fdb4ffa997e5f798bf8afe6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      2.708425   20773964.1056 0.1       \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEOCAYAAACTqoDjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYXFWd//H3t/e9O71m7XT2BRIgCxACkiigyCIKgsugMiriCso4/tSZUUbncRxGxgUREBQRV2SRRXaBAAlIQtbuQPbQSXfSa3rfqur8/qhKaEJ1d/VSfaurP6/nqadv3Xvr1vf07a5vnXvOPcecc4iIiBwvwesAREQkNilBiIhIWEoQIiISlhKEiIiEpQQhIiJhKUGIiEhYShAiIhKWEoSIiISlBCEiImEpQYiISFhJXgcwWIWFha6srMzrMERExpQNGzbUOeeKBvOaMZcgysrKWL9+vddhiIiMKWa2f7Cv0SUmEREJSwlCRETCUoIQEZGwlCBERCQsJQgREQlLCUJERMIac91cRWR8c86x+UATBhRlp5KUYBRkpZKYYMf26ezx4w84MlP1ETcc+u2JyJhy+5o9/OCx19+2rqwgg//64CIqG9q5f+NBNr15BIfjtBkFLCubwNySbEpyUplekElhVqpHkY89ShAiMmY8XXGY/378dc4/cSIfWjKV+tYuOnr83PniXj5+xysAzCvJ5qqVZQD8/fUafvLMTpwLvj7B4F1zi7hs6VTOWVBCWnKiRyUZG8wd/c2NEcuWLXO6k1pk/Klv7WL1/z7H9IJM/vy5FaSnvPXh3tbl4y8bDjBvYjanzcjH7K3LTe3dPvbUtlHX2sX6fY3c99oBqps6yUxJJCc9mcKsVP7jooUsL8v3olgRaero4WfP7GTVvGLOnFM4pGOY2Qbn3LLBvEY1CBEZE2584g3au/383xUnvS05AGSmJvHJM8rCvi4jJYkTp+QCsGpeMV89dy5rd9fxRPkhun0B1u6u5/Lb1vGxU0v58rvnMDE3LdpFiVhtSxf3v3aA29bsobG9m7yM5CEniKFQghCRmNXZ4+dPr1bS1u3jT+sr+fTKGcwuzh7WMRMTjLPmFHHWnOC4dW1dPm584g3ueXk/9244wAdPnsInzyhj4eSckSjCkN29bh//+XAFvoBjxcwCvn3BgmOJbrToEpOIxKzbnt99rEG6JCeVp752NjlpyVF5r8qGdm59fjf3v3aQjh4/5y4s4frz5jJ/4ugmCn/A8fNnd3HTUzt4z/xivvn++cNOijC0S0xKECISk7p8fs764bPMLs7iJx85hYyUxFHpttrU3sNv1u3jl2v20Nrt48LFk/n6efMoLciI+ntv2N/Avz1YzvbqZi45eTI3fvgkkhNH5nY1tUGISNz468Yqalq6+NHlJ1GUPXpdU3MzkvnKe+bwiRXT+eULe/j1S/t4quIQ150zl0+smE5GSnQ+Nt+sb+eTv3qV3PRkfvKRk7lo8WQSet3b4QXVIEQk5jjnOOem50lLTuSRL5/5tl5Jo626qYP/+Gs5T1UcJis1icuXTeNr580lawRrMz3+AJfduo49ta08du1ZTJ0w8rUV1SBEJC5s2N/I7to2brxssafJAWBSbjq3X7mUDfsb+d0rb/LrtXt5suIQ/3PZYs6YNbgeRYGAIyHB8Accr+ypp6G9m45uP79Zt49tB5v5+ceWRCU5DJUShIjEnPs3HiQtOYHzF03yOhQAzIxlZfksK8vnn04v5fo/b+Zjv3yFT6yYzqp5RZTkpDG3JLvf9oJ711fy73/dRlF2Kn6/o6qp89i2soIMbrr8JC5YHBvlPUoJQkRiSpfPz6NbqnnvCRNH9DLOSFk6PZ/Hrn0XP3z8de5au4+71wVn8kxNSqAoOxXnCI0Dlcg/nzmD80+cxN9fr+Eb921hSekESnLS6PIF+NYFC5hbkk23L8CCSTlvG0sqVsTeb19ExrVnX6+hqaOHDy2Z6nUofUpPSeS7F5/A51fNorqpk8qGdjZXHqGhvZsEMxLN2FHTwrcf2Ma3H9gGwNLpE7j706dGrZE7GsZOpCIS95o6evjJM7soyk5l5awCr8MZUElOGiU5aZw8LY+LTpr8tm3OOZ7bUcuuw62UFWZy1pzCMTf2kxKEiHjur5sO0tjWzQMbD7KrpoXbr1xG0gj1//eKmbF6XjGr5xV7HcqQKUGIiKf21bVx7R83AZCSlMAtH1/K6vlj90M1nihBiIinthxsAuAPnz2dRVNzY7JherzSmRART2072ERKYgJLp08gJWlsX1aKNzobIuKprQeamD8pW8khBumMiIhnAgHHtoNNLBrlYawlMkoQIuKZ/Q3ttHT5lCBilBKEiHhma6iBerQnwpHIKEGIiGeONlDPLRn+hDgy8pQgRMQTG/Y38ret1WqgjmE6KyIy6u5/7QCX/mItPf4A33jffK/DkT7oPggRGXV3vriX+ROzue/zZ4zKNKIyNKpBiMioev1QM+VVzVyxfJqSQ4yLWoIws2lm9qyZVZhZuZldG2afj5vZFjPbamZrzeykaMUjIrHhvg0HSEowLj5u9FOJPdFM3z7geufca2aWDWwws6eccxW99tkLnO2cazSz84HbgdOiGJOIeMjnD/DAxipWzy+mICvV63BkAFFLEM65aqA6tNxiZtuBKUBFr33W9nrJy0DszhAiIsP28JYq6lq7+PBS/auPBaPSBmFmZcApwCv97PZp4LHRiEdERl+3L8BNT+1g4aQczllQ4nU4EoGotxCZWRZwH3Cdc665j31WE0wQZ/ax/WrgaoDS0tIoRSoi0fSn9ZVUNnTw66tOJCEG51+Wd4pqDcLMkgkmh9855+7vY5/FwB3AB5xz9eH2cc7d7pxb5pxbVlRUFL2ARSQqun0Bbv77Tk4ty2fVXP0PjxXR7MVkwJ3AdufcTX3sUwrcD1zpnNsRrVhExFuPbq3icHMXX3z3bIIfDTIWRPMS00rgSmCrmW0KrfsWUArgnLsV+A+gALgl9Efjc84ti2JMIjLKnHPc8cJe5hRn8a45hV6HI4MQzV5MLwL9flVwzn0G+Ey0YhAR772yt4HyqmZ+8KFFqj2MMbqTWkSixucP8IPHXqcgM4UPnjLF63BkkHSfu4hEzW1r9rC58gg/++gppCUneh2ODJJqECISFTsPt/Djp3dwweJJXKRhNcYkJQgRiYqfP7uL5MQE/vPiE7wORYZICUJERlxlQzsPb6nm46eVasylMUwJQkRG3G1rdpNg8OkzZ3odigyDEoSIjKialk7+vP4Aly6ZysTcNK/DkWFQghCREfWrF/fh8wf43NmzvA5FhkkJQkRGTFNHD/e8vJ/zF01iRmGm1+HIMClBiMiIuefl/bR2+fi8ag9xQQlCREbMkxWHWTp9AidOyfU6FBkBShAiMiK6fQG2VzezpDTP61BkhChBiMiI2HG4hW5fgMVTlSDihRKEiIyILQeaADhJCSJuKEGIyIjYcuAIeRnJTMtP9zoUGSFKECIyIrYcaGLRlFzN+RBHlCBEZNg6e/y8cbhFl5fijBKEiAxbRXUz/oBj0VR1b40nShAiMmxbQw3Ui5Ug4ooShIgMW0VVMwWZKUzM0eB88UQJQkSGraK6mYWTc9RAHWeUIERkWHr8Ad441MLCSTlehyIjTAlCRIZld20r3f4ACycrQcQbJQgRGZaKqmYATlCCiDtKECIyLOVVzaQlJzCjMMvrUGSEKUGIyLBUVDUzb2IOiQlqoI43ShAiMmTOOSqqm3V5KU4pQYjIkB1u7qKpo4cFE7O9DkWiYMAEYWaZZpYQWp5rZhebWXL0QxORWPdmQzsApQWafzoeRVKDWAOkmdkU4EngSuCuaAYlImNDZShBTJugIb7jUSQJwpxz7cCHgFuccx8GTohuWCIyFlQ2tmMGU5Qg4lJECcLMVgAfBx4NrUuMXkgiMlZUNnQwMSeN1CR9JMSjSBLEdcA3gQecc+VmNhN4NrphichYUNnYzrQJGV6HIVEyYIJwzj3vnLvYOffDUGN1nXPuKwO9zsymmdmzZlZhZuVmdm2YfczMfmpmu8xsi5ktGWI5RMQDlQ3tTNUUo3Erkl5MvzezHDPLBLYBFWb29QiO7QOud84tBE4HvmhmC4/b53xgTuhxNfCLQUUvIp7p8vk51NypGkQci+QS00LnXDNwCfAYMINgT6Z+OeeqnXOvhZZbgO3AlON2+wBwtwt6Gcgzs0mDKYCIeKPqSCfOwbR8JYh4FUmCSA7d93AJ8JBzrgdwg3kTMysDTgFeOW7TFKCy1/MDvDOJiEgMUhfX+BdJgrgN2AdkAmvMbDrQHOkbmFkWcB9wXagmMmhmdrWZrTez9bW1tUM5hIiMsMrGUIJQDSJuRdJI/VPn3BTn3PtDl4L2A6sjOXio5nEf8Dvn3P1hdjkITOv1fGpo3fEx3O6cW+acW1ZUVBTJW4tIlFU2dJCSmECJphmNW5E0Uuea2U1Hv8Gb2Y8I1iYGep0BdwLbnXM39bHbQ8AnQr2ZTgeanHPVgymAiHijsrGdKRPSNYprHEuKYJ9fEey9dHno+ZXArwneWd2flaF9t5rZptC6bwGlAM65W4G/Ae8HdgHtwFWDCV5EvLO/vk2Xl+JcJAlilnPu0l7Pb+j1gd8n59yLQL9fLZxzDvhiBDGISAzx+QPsONzKJ1cUeB2KRFEkjdQdZnbm0SdmthLoiF5IIhLr9tS10e3TPNTxLpIaxOeB35hZLsEaQQPwqWgGJSKxbXt1sEPigklKEPFswAThnNsEnGRmOaHnQ+qqKiLxo6KqmZTEBGYVaR7qeNZngjCzr/WxHoB+eiaJSJyrqG5mTkkWyYmalDKe9VeD0ByCIhLW9upmVs8r9joMibI+E4Rz7obRDERExoaalk7qWrvV/jAOqH4oIoNSURVshlQPpvinBCEig7LlQBMACyYqQcQ7JQgRGZTHth3ilNI8cjOSvQ5FomzAbq5mlgpcCpT13t8595/RC0tEYtGumha2VzfznYuOn/tL4lEkN8r9FWgCNgBd0Q1HRGLZQ5uqSDC4YLHm9RoPIkkQU51z74t6JCIS05xzPLS5ihWzCijO1hDf40EkbRBrzWxR1CMRkZj20q569tW3c9HiyV6HIqMkkhrEmcCnzGwvwUtMRnAg1sVRjUxEYkaPP8AND5czLT+dS07RrMDjRSQJ4vyoRyEiMe3udfvZWdPK7VcuJS050etwZJREMuXofiAPuCj0yAutE5FxoNsX4Oa/7+SsOYWcu7DE63BkFEUy5ei1wO+A4tDjHjP7crQDE5HY8PyOWhrbe7hqZdmxwTplfIjkEtOngdOcc20AZvZDYB3ws2gGJiKx4cFNB8nPTOGsOUVehyKjLJJeTAb4ez33M8BUoiISH1o6e3i64jAXLp6kob3HoUhqEL8GXjGzB0LPLwHujF5IIhIrnig/TJcvwAdOVs+l8SiSGeVuMrPnCHZ3BbjKObcxqlGJSEx4fFs1U/LSWVKa53Uo4oH+ZpTLcc41m1k+sC/0OLot3znXEP3wRMQrXT4/a3fX86ElU9Q4PU71V4P4PXAhwTGYXK/1Fno+M4pxiYjH1u9rpL3bz6q5mjluvOpvRrkLQz9njF44IhIrnnujhpTEBM6YXeB1KOKRSO6DeCaSdSISX557o5ZTZ+STkRJJXxaJR/21QaQBGUChmU3gra6tOYC6NIjEsYNHOthZ08oVy6d5HYp4qL+vBp8DrgMmE2yHOJogmoGboxyXiHjoH3vrAVg5u9DjSMRL/bVB/AT4iZl92Tmnu6ZFxpHyg82kJiUwpzjL61DEQ5HcB/EzMzsRWAik9Vp/dzQDExHvlFc1M39iNkm6e3pci2RO6u8AqwgmiL8RHP77RUAJQiQOOecor2riAk0MNO5F8vXgMuA9wCHn3FXASUBuVKMSEc8caOygudPHCZNzvA5FPBZJguhwzgUAn5nlADWAujaIxKnyqmYAJQiJaLC+9WaWB/ySYG+mVoLDfYtIHKqoaiLBYP5EJYjxLpJG6i+EFm81s8eBHOfcloFeZ2a/IjhUR41z7sQw23OBe4DSUBz/65z79WCCF5GRV17VzKyiLNJTNLXoeNffjXJL+tvmnHttgGPfRfB+ib4as78IVDjnLjKzIuANM/udc657gOOKSBSVVzVz+sx8r8OQGNBfDeJHoZ9pwDJgM8Gb5RYD64EV/R3YObfGzMr62wXItuAwkVlAA+CLKGoRiYqmjh4ONXcyf5IuL0k/jdTOudXOudVANbDEObfMObcUOAU4OALvfTOwAKgCtgLXhhrDRcQje2pbAZhZmOlxJBILIunFNM85t/XoE+fcNoIf7MP1XmATwaE8TgZuDvWSegczu9rM1pvZ+tra2hF4axEJZ09tGwCzdAe1EFmC2GJmd5jZqtDjl8CAjdQRuAq43wXtAvYC88Pt6Jy7PVSDWVZUpInTRaJld20rSQlGaX6G16FIDIgkQVwFlAPXhh4VoXXD9SbBG/AwsxJgHrBnBI4rIkO0p7aN0vwMkjXEhhBZN9dO4P9Cj4iZ2R8IDtFRaGYHgO8AyaFj3gp8D7jLzLYSbPz+hnOublDRi8iI2lPXyswitT9IUH/dXP/snLs89AHujt/unFvc34Gdcx8dYHsVcF6kgYpIdPkDjn317ayepylGJai/GsS1oZ8XjkYgIuKtg40ddPsCqkHIMf3NB1Ed+rl/9MIREa/srgt1cS1SDyYJ6u8SUwthLi0RbC9wzjndSSMSR3bX6B4Iebv+ahDZoxmIiHhrT10beRnJ5GemeB2KxIhIRnMFwMyKefuMcm9GJSIR8cTumlZmFmYSHP1GJIL7IMzsYjPbSfBGtueBfcBjUY5LREbZzppW5pbowoG8JZK7Yb4HnA7scM7NIHhz28tRjUpERlVdaxcNbd3MUYKQXiJJED3OuXogwcwSnHPPEhzdVUTixI7DLQDMLVEPJnlLJG0QR8wsC1gD/M7MaoC26IYlIqNp5+FgDyZdYpLeIqlBfABoB74KPA7sBi6KZlAiMrp21rSQnZZEcXaq16FIDImkBvE54E/OuYPAb6Icj4h4YMfhYAO1ejBJb5HUILKBJ83sBTP7UmjkVRGJE845dh5uUfuDvMOACcI5d4Nz7gSCc0hPAp43s6ejHpmIjIq61m4a23uYU6z2B3m7wQz6XgMcAuoBDfcoEid2hnowzVENQo4TyY1yXzCz54BngALgswMN9S0iY8fOGvVgkvAiaaSeBlznnNsU7WBEZPTtONxCjnowSRiRzCj3zdEIRES8sVM9mKQPmnhWZBxzzrGjpkVDbEhYShAi41htaxdH2nuYU6wGanknJQiRcWyXhtiQfihBiIxjGqRP+qMEITKO7ahpJTc9mSL1YJIwlCBExrGdh1uYU5ylHkwSlhKEyDjlnGPH4Vb1YJI+KUGIjFNvNrTT1NHDPLU/SB+UIETGqQc3VgFwzkIN0CzhKUGIjEPOOe577QBnzCpg6oQMr8ORGKUEITIOvbqvkTcb2rl0yVSvQ5EYpgQhMg7dt+EAmSmJnL9ootehSAxTghAZZwIBx9PbD3POwhIyUiIZ0FnGKyUIkXHmjcMt1Ld1c+bsQq9DkRinBCEyzry0qw6AlUoQMgAlCJFx5qVddcwszGRyXrrXoUiMU4IQGUd6/AH+sbeBM2YXeB2KjAFRSxBm9iszqzGzbf3ss8rMNplZuZk9H61YRCRoc+UR2rr9rJyly0sysGjWIO4C3tfXRjPLA24BLnbOnQB8OIqxiAjw0q56zGDFLNUgZGBRSxDOuTVAQz+7fAy43zn3Zmj/mmjFIiJBL+2q44TJOeRlpHgdiowBXrZBzAUmmNlzZrbBzD7hYSwica+928fGykb1XpKIeXmXTBKwFHgPkA6sM7OXnXM7jt/RzK4GrgYoLS0d1SBF4sU/9jbQ43dqf5CIeVmDOAA84Zxrc87VAWuAk8Lt6Jy73Tm3zDm3rKioaFSDFIkXa3fXk5KYwPKyfK9DkTHCywTxV+BMM0syswzgNGC7h/GIxLUXd9ZxSmke6SmJXociY0TULjGZ2R+AVUChmR0AvgMkAzjnbnXObTezx4EtQAC4wznXZ5dYERm6utYuKqqbuf7cuV6HImNI1BKEc+6jEexzI3BjtGIQkaCb/76LBIP3nqjRWyVyupNaJM5tr27m7nX7+Php05mr+adlEJQgROLc9x+tIDc9mevP0+UlGRwlCJE41tjWzdrd9XzyjDLdHCeDpgQhEsde3lOPc3DWHN37IIOnBCESx17aXUdmSiKLp+Z5HYqMQUoQInFs7a56Tp2RT3Ki/tVl8PRXIxKnqps62FPXprGXZMiUIETi1Npd9QCcobGXZIiUIETi1As7a8nPTGH+RN37IEOjBCESh3r8AZ59o5bV84pJSDCvw5ExysvhvkdVU3sPt67ZzXXnzCE1KThY2YHGdn67bj+pyYm0dfl4fkctc4qz+K8PLiI/8+19xnfXtlKYmUpuRrIX4YsMyvp9jTR19HDuwhKvQ5ExbNwkiOd21PCL53azufII37vkRJ59vYb/e2oHXb4AfudITkhgWdkEntlew2tvruGas2dxzoISunx+bnl2N/dvPEhqUgKr5xWTnZZEUmICOelJzJ+YzcpZhRTnpHldRJFjnqo4TEpSgu5/kGEZNwniAydPwR9w/OtftvCeHz0PwMrZBfzw0sVMzEnDF3CkJSdSXtXEt+7fyg0PV3DDwxUAJCUY15w9i7YuH8++UYM/4OjxB2ju8NHtDwBwzoISPr9qJkuna6x98ZZzjqe2H2LlrAIyU8fNv7hEwbj66/nQkqmU5mewqfIIZ84pZF5JNmbB67Ohq06cMDmXv37pTF4/1Mz6fY1kpgZvMppVlPWO4wUCjorqZp4sP8TdL+/n0l8cZtn0CVy+fBqnluUzvSDj2PFFRssbh1uobOjg82fP9joUGePMOed1DIOybNkyt379eq/DeIf2bh9/frWSX76wl4NHOgCYWZTJOQtKKM3PYEnpBBZOzgGgqaOHNw61kJRonDItT0lERtTX793MQ5ureOn/vZvCrFSvw5EYYWYbnHPLBvOacVWDiKaMlCQ+tXIGn1hRxu7aVtbtqefxbYe488W9+APBJPye+cUc6ehhw/7GY6+bV5LNh5dN5b0nTGRafoZX4UucqGxo54GNB/mn06crOciwqQYRZf6A43BzJ/euP8Cv1+5lUm467zthIoun5VLT3Mk9L7/J1oNNAJTmZ7B4ai5T8tKpbGzn1X2NJCUYacmJ1Ld2MSEzhXMWlHDajHxOmpZHiRrG5TjfemArf1l/gDX/upqJufr7kLcMpQahBBED9te38VTFYV7d10BFdTOHm7rIz0xhxawCEhOMzh4/+ZkpVDa089Ku+mMN46eU5nH23CIm5aaxvCyfmWHaSWT8qDrSwdk3PssVy6fx/UsWeR2OxBhdYhqjphdk8pmzZvKZs2YCwV4ofbVLdHT7qahu4pW9DTyyuZofP73z2LZTSvNYNn0CJ03L49yFJcfu9xiIzx+g6kgnRdmpmtB+DLvt+d04B9ecPcvrUCROKEHEoP4ardNTElk6PZ+l0/P5wqrZdPb4qW7q5InyQ/xtazW/Wbef7hf2UpiVwup5xUwvyCAtORFfwHGkvYes1EQWTc2jvctHRXUzL+6qo/xgM93+ALnpyXz01FI+sWI6k/PSR7HEMlw1zZ384dVKLl0ylakT1JYlI0OXmOKMzx/gpd313PPyfjZVHqG2pevYtuREo8f/1vlOMFg0NY/TZuRTVpDJCztreaL8EGbGktI8unwBZhZm8k+nT2fp9AnqbRWjfP4A/3LvZh7eUs3frz+b6QWZXockMUhtEPIOnT1+evwBEhOM9OREmjt9lFc1kZOWzIzCzHfcSFXZ0M7d6/bx2ptHyEhJZFPlEVo6fSyYlMPHTp3G6TMLSEtOZMfhFnLTk5lTnK3hRzzU0NbNl37/Gmt31/OVd8/ma+fN8zokiVFKEDLi2rt9PLixirvX7eP1Qy1h95lXks2S6ROYW5LFSdPyWDwllyRNUBN1FVXNXP3b9dS0dPFfl5zIh5dN8zokiWFKEBI1zjl217ax8c1GevyOuSVZNHf2UFHVzCt7G9hyoImmjh4AslKTOHVGPmfNKeSCRZM0TlUUvFnfzvt+soactGRuvXIpJ0/TlKLSPyUI8YxzjtqWLl7d18ja3XWs213Pnro2zGBOcRZT8tKZnJfOjMLg3eVlhbpOPhzX/HYDz++o5enrz2aKOhRIBNTNVTxjZhTnpHHB4klcsHgSALtqWnl4cxUV1c1UHelgU+URGtt7+P6j2zm1LJ9/fd88lpVpcMPBWre7nsfLD3H9uXOVHCSqVIOQUXXwSAePbK7ijhf3UtvSxbkLS/iX8+YxqyhT7RYD6Ozxc8/L+7nlud2kJyfyzPVnk5as+1YkMrrEJGNGe7ePO1/Yy21r9tDa5QMgMyWR7LRkunx+Wjp9zCnJ5rQZ+Vy2dConTsn1OGJvba48wvX3bmZXTSsrZxfw7xcuZP7EHK/DkjFECULGnPrWLh7dWk1DWzfNHT5aOntIS04kPSWR7dXBBvBuX4Ci7NRjN+8lJxjLZ+Rz1uxClpZNiPiO8bGo2xfgp8/s5BfP76YoK5UfXLqI1fOKvQ5LxiAlCIk7Te09PLT5IFsPNlHd1ImZ0drZw5YDTaFJnhI4dUYBK2cVcErpBE6ckkNGSnw0rR1obOdzv91AeVUzly2dyr9fuJDcdN1zIkOjRmqJO7kZyVy5ouwd61u7fLyyp54Xdtbx4q46fvDY6wCkJiXwrrlFzJ+YTWZqEucsKGZ2cfYoRz08bV0+nig/xPcf3U6PP8DtVy7lvBMmeh2WjEOqQUhcqG3pYsuBI7yws44nyw9R3dzJ0T/tEybnsHBSDmfMLuD8Eyfx0q46nt9RS256MlPy0pk7MZvi7FSy05LJTk3CDLr9ATq6/XT2BCjISiF5lBrQnyw/xFf/tIm2bj9zirO49cqlYWczFBksXWISCXHOUdvaxYMbD/Ls67XsrGmhrrWblMQEuv0B0pMT6fL5CYT5809KMHyBt49ZVVaYyXtPmMgFiyZxwuScER+Xqscf4OmtQhGrAAAK6UlEQVSKw3zljxtZOCmHb71/AcvL8klI0PhXMjJiKkGY2a+AC4Ea59yJ/ey3HFgHfMQ595eBjqsEIUPhnOOlXfX8bVs1y8smcOHiySSYcbCxgx2HW6hv66Kl00dzpw+fP0BmahIZKYkkJyZQ09zJxsojrN1djz/gmF6QwYlTcplVmMkp0yeQlZrE69XNHGru5Eh7D0kJRlZaEiU5aRRnp1GUnUJ7tx+ABZNyjs30treujb9sqOTFXfVUVDXR43csmpLLPZ85TW0NMuJiLUG8C2gF7u4rQZhZIvAU0An8SglCYlljWzdPVhziqYrD7KpppbKx49h0sgCJCUZuejL+gKO1y/e2bb1Nzk1jWn4Gr+5rIMGMk6flsawsn3kTszh34USyUtU0KCMvphqpnXNrzKxsgN2+DNwHLI9WHCIjZUJmClcsL+WK5aVA8F6OTW8eodPnZ8GkHEqy045dEvIHHPVtXdQ0d1HX2kVGShI+f4Dyqma2HGxiV00rnz1rJp8+awbF2RqrSmKTZ19VzGwK8EFgNUoQMgZlpCRxxuzCsNsSE4zi7LR3fPj3tb9ILPJybIMfA99wzgUG2tHMrjaz9Wa2vra2dhRCExERLy92LgP+GOoNUgi838x8zrkHj9/ROXc7cDsE2yBGNUoRkXHKswThnJtxdNnM7gIeCZccRETEG1FLEGb2B2AVUGhmB4DvAMkAzrlbo/W+IiIyMqLZi+mjg9j3U9GKQ0REhkYD8IuISFhKECIiEpYShIiIhDXmBuszs1pg/xBfngs0DXO/cNsGWnf89qPPe68vBOoiiK0/kZRvsGULt76/530tD7d8sXbuei+P1rnrbz+du/DrBipfLJy7vraN9Lmb7pwriiDOtzjnxs0DuH24+4XbNtC647cffX7cPutHo3yDLVt/8Q9U1pEsX6ydu+PO46icu8GcI527yMoXC+duKGUZrXM33i4xPTwC+4XbNtC647c/3Mf64YrkeIMtW7j1/T3vr9zDEWvnbjAxRWK45dO5C78unso36mUbc5eY4pWZrXeDHGlxLInn8sVz2SC+yxfPZYPhl2+81SBi2e1eBxBl8Vy+eC4bxHf54rlsMMzyqQYhIiJhqQYhIiJhKUGIiEhYShAiIhKWEsQYYWaZoUmTLvQ6lpFkZgvM7FYz+4uZfd7reEaamV1iZr80sz+Z2XlexzOSzGymmd1pZgPOJT9WhP7PfhM6Zx/3Op6RNJTzpQQRZWb2KzOrMbNtx61/n5m9YWa7zOz/RXCobwB/jk6UQzMSZXPObXfOXQNcDqyMZryDNULle9A591ngGuCKaMY7GCNUtj3OuU9HN9LhG2RZPwT8JXTOLh71YAdpMGUbyvlSgoi+u4D39V5hZonAz4HzgYXAR81soZktMrNHjnsUm9m5QAVQM9rBD+Auhlm20GsuBh4F/ja64Q/oLkagfCH/FnpdrLiLkStbrLuLCMsKTAUqQ7v5RzHGobqLyMs2aF5OOTouOOfWmFnZcatPBXY55/YAmNkfgQ84534AvOMSkpmtAjIJnuwOM/ubi2Au72gbibKFjvMQ8JCZPQr8PnoRD84InTsD/ht4zDn3WnQjjtxInbuxYDBlBQ4QTBKbGANfoAdZtorBHj/mfwFxagpvfUuB4B/llL52ds592zl3HcEPz1/GQnLox6DKZmarzOynZnYbsVeDCGdQ5QO+DJwDXGZm10QzsBEw2HNXYGa3AqeY2TejHdwI66us9wOXmtkvGPmhcEZL2LIN5XypBjGGOOfu8jqGkeacew54zuMwosY591Pgp17HEQ3OuXqCbStxwznXBlzldRzRMJTzpRqENw4C03o9nxpaFw/iuWwQ3+WL57IdL57LOmJlU4LwxqvAHDObYWYpwEeAhzyOaaTEc9kgvssXz2U7XjyXdcTKpgQRZWb2B2AdMM/MDpjZp51zPuBLwBPAduDPzrlyL+McinguG8R3+eK5bMeL57JGu2warE9ERMJSDUJERMJSghARkbCUIEREJCwlCBERCUsJQkREwlKCEBGRsJQgJOrMrHUU3uPigYanjsJ7rjKzM4bwulPM7M7Q8qfM7OaRj27wzKzs+GGjw+xTZGaPj1ZM4i0lCBkzQsMYh+Wce8g5999ReM/+xitbBQw6QQDfYoyOz+ScqwWqzSym5u6Q6FCCkFFlZl83s1fNbIuZ3dBr/YNmtsHMys3s6l7rW83sR2a2GVhhZvvM7AYze83MtprZ/NB+x76Jm9ldoRFi15rZHjO7LLQ+wcxuMbPXzewpM/vb0W3Hxficmf3YzNYD15rZRWb2ipltNLOnzawkNMTyNcBXzWyTmZ0V+nZ9X6h8r4b7EDWzbGCxc25zmG1lZvb30O/mGTMrDa2fZWYvh8r7/XA1MgvOhPaomW02s21mdkVo/fLQ72Gzmf3DzLJD7/NC6Hf4WrhakJklmtmNvc7V53ptfhCIq9nWpA/OOT30iOoDaA39PA+4HTCCX04eAd4V2pYf+pkObAMKQs8dcHmvY+0Dvhxa/gJwR2j5U8DNoeW7gHtD77GQ4Nj4AJcRHFI8AZgINAKXhYn3OeCWXs8n8NaoA58BfhRa/i7wL732+z1wZmi5FNge5tirgft6Pe8d98PAJ0PL/ww8GFp+BPhoaPmao7/P4457KcGh4I8+zwVSgD3A8tC6HIIjOGcAaaF1c4D1oeUyYFto+Wrg30LLqcB6YEbo+RRgq9d/V3pE/6HhvmU0nRd6bAw9zyL4AbUG+IqZfTC0flpofT3BWb3uO+4494d+biA4RWQ4D7rgvBkVZlYSWncmcG9o/SEze7afWP/Ua3kq8Cczm0TwQ3dvH685B1hoZkef55hZlnOu9zf+SUBtH69f0as8vwX+p9f6S0LLvwf+N8xrtwI/MrMfAo84514ws0VAtXPuVQDnXDMEaxvAzWZ2MsHf79wwxzsPWNyrhpVL8JzsJTiz4eQ+yiBxRAlCRpMBP3DO3fa2lcEZ884BVjjn2s3sOSAttLnTOXf81I9doZ9++v4b7uq1bH3s05+2Xss/A25yzj0UivW7fbwmATjdOdfZz3E7eKtsI8Y5t8PMlgDvB75vZs8AD/Sx+1eBw8BJBGMOF68RrKk9EWZbGsFySJxTG4SMpieAfzazLAAzm2LBuY1zgcZQcpgPnB6l93+J4GxhCaFaxaoIX5fLW+Ppf7LX+hYgu9fzJwnOIAdA6Bv68bYDs/t4n7UEh2aG4DX+F0LLLxO8hESv7W9jZpOBdufcPcCNwBLgDWCSmS0P7ZMdanTPJVizCABXAuEa/58APm9myaHXzg3VPCBY4+i3t5PEByUIGTXOuScJXiJZZ2Zbgb8Q/IB9HEgys+0E529+OUoh3Edw+sUK4B7gNaApgtd9F7jXzDYAdb3WPwx88GgjNfAVYFmoUbeCMLN3OedeB3JDjdXH+zJwlZltIfjBfW1o/XXA10LrZ/cR8yLgH2a2CfgO8H3nXDdwBfCzUCP/UwS//d8CfDK0bj5vry0ddQfB39Nroa6vt/FWbW018GiY10ic0XDfMq4cbRMwswLgH8BK59yhUY7hq0CLc+6OCPfPADqcc87MPkKwwfoDUQ2y/3jWAB9wzjV6FYOMDrVByHjziJnlEWxs/t5oJ4eQXwAfHsT+Swk2KhtwhGAPJ0+YWRHB9hglh3FANQgREQlLbRAiIhKWEoSIiISlBCEiImEpQYiISFhKECIiEpYShIiIhPX/AaUuqg51mk+aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "lr=1e-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "755912eb710041aa839773759ffef745",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.170261   1.077664   0.6091    \n",
      "    1      1.030724   0.958806   0.6595                     \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.9588064260005951, 0.6595]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 2, cycle_len=1, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d294526e27b04db885860868e25522f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.940711   0.886267   0.6889    \n",
      "    1      0.942904   0.877985   0.6915                      \n",
      "    2      0.751138   0.768557   0.7296                      \n",
      "    3      0.85915    0.939773   0.6852                      \n",
      "    4      0.716295   0.789837   0.73                        \n",
      "    5      0.606361   0.663592   0.7682                      \n",
      "    6      0.53345    0.642123   0.7743                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.6421226511478424, 0.7743]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 3, cycle_len=1, cycle_mult=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.save('16x16_8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## 24x24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.load('16x16_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.set_data(get_data(24,bs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbd2ee85744040f885a0c291ff4086b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.675646   0.783434   0.7319    \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.7834340079784393, 0.7319]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(1e-2, 1, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68edf51ef9744ad9bb4b9f7846ee8fc8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.635211   0.606865   0.7913    \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.6068654565572739, 0.7913]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 1, cycle_len=1, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8694ef954f89422c9cd4d01ac206fcdc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.5435     0.531886   0.8182    \n",
      "    1      0.561118   0.569581   0.8091                      \n",
      "    2      0.406358   0.45027    0.849                       \n",
      "    3      0.530289   0.766036   0.7569                      \n",
      "    4      0.427189   0.509906   0.8334                      \n",
      "    5      0.30009    0.384696   0.8727                      \n",
      "    6      0.240064   0.373127   0.8749                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.3731268438577652, 0.8749]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 3, cycle_len=1, cycle_mult=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn.save('24x24_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                             \r"
     ]
    }
   ],
   "source": [
    "log_preds,y = learn.TTA()\n",
    "#preds = np.mean(np.exp(log_preds),0), metrics.log_loss(y,preds), accuracy_np(preds,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 32x32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('24x24_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.set_data(get_data(32,bs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66a7594cc07a45c3bc80cae6de92d4fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.31825    0.38657    0.8673    \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.3865702114582062, 0.8673]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(1e-2, 1, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "864f905943ce480e8a8300b2baab1b66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/391 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (2) : out of memory at /pytorch/aten/src/THC/generic/THCStorage.cu:58",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-07f7eca2b8d3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcycle_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcycle_mult\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/learner.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lrs, n_cycle, wds, **kwargs)\u001b[0m\n\u001b[1;32m    302\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cycle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarm_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    249\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 251\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    252\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, visualize, **kwargs)\u001b[0m\n\u001b[1;32m    139\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_stepper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m             \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mavg_mom\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mavg_mom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m             \u001b[0mdebias_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mavg_mom\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/model.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, xs, y, epoch)\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0mxtra\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxtra\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/models/cifar10/resnext.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn_1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstage_1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstage_2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     98\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstage_3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mavgpool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     89\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai-0.7.0-py3.6.egg/fastai/models/cifar10/resnext.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m     \u001b[0mbottleneck\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbottleneck\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m     \u001b[0mbottleneck\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbottleneck\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0mbottleneck\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv_expand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbottleneck\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/py3env/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mrelu\u001b[0;34m(input, inplace)\u001b[0m\n\u001b[1;32m    616\u001b[0m     \"\"\"\n\u001b[1;32m    617\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 618\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    619\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    620\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (2) : out of memory at /pytorch/aten/src/THC/generic/THCStorage.cu:58"
     ]
    }
   ],
   "source": [
    "learn.fit(lr, 3, cycle_len=1, cycle_mult=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lr, 3, cycle_len=4, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_preds,y = learn.TTA()\n",
    "metrics.log_loss(y,np.exp(log_preds)), accuracy_np(log_preds,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('32x32_8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "nav_menu": {
    "height": "266px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
